2024-11-11 16:14:01,151 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 16:14:01,178 main DEBUG Took 0.023062 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 16:14:01,179 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 16:14:01,179 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 16:14:01,181 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 16:14:01,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,202 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 16:14:01,226 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,228 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,229 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,229 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,230 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,230 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,231 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,232 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,233 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,244 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,269 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,277 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,281 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,282 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,283 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 16:14:01,284 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,285 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 16:14:01,294 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 16:14:01,297 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 16:14:01,299 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 16:14:01,300 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 16:14:01,302 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 16:14:01,302 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 16:14:01,316 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 16:14:01,319 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 16:14:01,323 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 16:14:01,324 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 16:14:01,324 main DEBUG createAppenders(={Console}) 2024-11-11 16:14:01,325 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 16:14:01,325 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 16:14:01,326 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 16:14:01,326 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 16:14:01,327 main DEBUG OutputStream closed 2024-11-11 16:14:01,327 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 16:14:01,327 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 16:14:01,328 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 16:14:01,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 16:14:01,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 16:14:01,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 16:14:01,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 16:14:01,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 16:14:01,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 16:14:01,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 16:14:01,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 16:14:01,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 16:14:01,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 16:14:01,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 16:14:01,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 16:14:01,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 16:14:01,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 16:14:01,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 16:14:01,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 16:14:01,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 16:14:01,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 16:14:01,694 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 16:14:01,695 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 16:14:01,695 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 16:14:01,696 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T16:14:02,054 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9 2024-11-11 16:14:02,057 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 16:14:02,057 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T16:14:02,067 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T16:14:02,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T16:14:02,158 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35, deleteOnExit=true 2024-11-11T16:14:02,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T16:14:02,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/test.cache.data in system properties and HBase conf 2024-11-11T16:14:02,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T16:14:02,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/hadoop.log.dir in system properties and HBase conf 2024-11-11T16:14:02,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T16:14:02,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T16:14:02,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T16:14:02,312 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T16:14:02,480 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T16:14:02,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T16:14:02,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T16:14:02,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T16:14:02,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T16:14:02,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T16:14:02,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T16:14:02,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T16:14:02,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T16:14:02,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T16:14:02,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/nfs.dump.dir in system properties and HBase conf 2024-11-11T16:14:02,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/java.io.tmpdir in system properties and HBase conf 2024-11-11T16:14:02,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T16:14:02,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T16:14:02,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T16:14:03,594 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T16:14:03,719 INFO [Time-limited test {}] log.Log(170): Logging initialized @3905ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T16:14:03,821 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:03,927 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T16:14:03,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T16:14:03,987 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T16:14:03,989 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T16:14:04,008 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:04,026 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69b57b46{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/hadoop.log.dir/,AVAILABLE} 2024-11-11T16:14:04,027 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65b5c633{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T16:14:04,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3042afd9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/java.io.tmpdir/jetty-localhost-40237-hadoop-hdfs-3_4_1-tests_jar-_-any-8219175024460310987/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T16:14:04,420 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7e4ef918{HTTP/1.1, (http/1.1)}{localhost:40237} 2024-11-11T16:14:04,420 INFO [Time-limited test {}] server.Server(415): Started @4607ms 2024-11-11T16:14:05,071 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T16:14:05,080 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T16:14:05,082 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T16:14:05,083 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T16:14:05,083 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T16:14:05,084 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70efab76{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/hadoop.log.dir/,AVAILABLE} 2024-11-11T16:14:05,085 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43a5eb8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T16:14:05,222 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7740a90{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/java.io.tmpdir/jetty-localhost-39687-hadoop-hdfs-3_4_1-tests_jar-_-any-13034553136565643514/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T16:14:05,223 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c91b148{HTTP/1.1, (http/1.1)}{localhost:39687} 2024-11-11T16:14:05,223 INFO [Time-limited test {}] server.Server(415): Started @5410ms 2024-11-11T16:14:05,300 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T16:14:05,963 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/data/data1/current/BP-856990112-172.17.0.2-1731341643287/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:05,968 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/data/data2/current/BP-856990112-172.17.0.2-1731341643287/current, will proceed with Du for space computation calculation, 2024-11-11T16:14:06,185 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T16:14:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c5de3c2684fa5e5 with lease ID 0x98e1d6a489a18216: Processing first storage report for DS-1a41b93d-1191-4ce0-ad40-f93116a298a6 from datanode DatanodeRegistration(127.0.0.1:34773, datanodeUuid=5299dada-c81c-4e5e-99e5-4d475f2a8702, infoPort=43639, infoSecurePort=0, ipcPort=44659, storageInfo=lv=-57;cid=testClusterID;nsid=209619138;c=1731341643287) 2024-11-11T16:14:06,318 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c5de3c2684fa5e5 with lease ID 0x98e1d6a489a18216: from storage DS-1a41b93d-1191-4ce0-ad40-f93116a298a6 node DatanodeRegistration(127.0.0.1:34773, datanodeUuid=5299dada-c81c-4e5e-99e5-4d475f2a8702, infoPort=43639, infoSecurePort=0, ipcPort=44659, storageInfo=lv=-57;cid=testClusterID;nsid=209619138;c=1731341643287), blocks: 0, hasStaleStorage: true, processing time: 14 msecs, invalidatedBlocks: 0 2024-11-11T16:14:06,320 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c5de3c2684fa5e5 with lease ID 0x98e1d6a489a18216: Processing first storage report for DS-fe6e86d2-7486-46b3-881d-37b77be3d258 from datanode DatanodeRegistration(127.0.0.1:34773, datanodeUuid=5299dada-c81c-4e5e-99e5-4d475f2a8702, infoPort=43639, infoSecurePort=0, ipcPort=44659, storageInfo=lv=-57;cid=testClusterID;nsid=209619138;c=1731341643287) 2024-11-11T16:14:06,320 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c5de3c2684fa5e5 with lease ID 0x98e1d6a489a18216: from storage DS-fe6e86d2-7486-46b3-881d-37b77be3d258 node DatanodeRegistration(127.0.0.1:34773, datanodeUuid=5299dada-c81c-4e5e-99e5-4d475f2a8702, infoPort=43639, infoSecurePort=0, ipcPort=44659, storageInfo=lv=-57;cid=testClusterID;nsid=209619138;c=1731341643287), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T16:14:06,350 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9 2024-11-11T16:14:06,583 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/zookeeper_0, clientPort=57646, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T16:14:06,598 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57646 2024-11-11T16:14:06,614 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:06,619 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:06,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741825_1001 (size=7) 2024-11-11T16:14:07,375 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 with version=8 2024-11-11T16:14:07,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/hbase-staging 2024-11-11T16:14:07,494 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T16:14:07,834 INFO [Time-limited test {}] client.ConnectionUtils(128): master/16b413a53992:0 server-side Connection retries=45 2024-11-11T16:14:07,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:07,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:07,854 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T16:14:07,854 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:07,854 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T16:14:08,049 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T16:14:08,126 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T16:14:08,138 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T16:14:08,143 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T16:14:08,183 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1037 (auto-detected) 2024-11-11T16:14:08,185 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T16:14:08,211 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33601 2024-11-11T16:14:08,240 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33601 connecting to ZooKeeper ensemble=127.0.0.1:57646 2024-11-11T16:14:08,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:336010x0, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T16:14:08,289 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33601-0x1002f9f05730000 connected 2024-11-11T16:14:08,353 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:08,357 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:08,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:14:08,375 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669, hbase.cluster.distributed=false 2024-11-11T16:14:08,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T16:14:08,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33601 2024-11-11T16:14:08,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33601 2024-11-11T16:14:08,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33601 2024-11-11T16:14:08,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33601 2024-11-11T16:14:08,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33601 2024-11-11T16:14:08,607 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b413a53992:0 server-side Connection retries=45 2024-11-11T16:14:08,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:08,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:08,610 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T16:14:08,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T16:14:08,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T16:14:08,614 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T16:14:08,617 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T16:14:08,618 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45121 2024-11-11T16:14:08,620 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45121 connecting to ZooKeeper ensemble=127.0.0.1:57646 2024-11-11T16:14:08,621 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:08,624 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:08,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451210x0, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T16:14:08,632 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45121-0x1002f9f05730001 connected 2024-11-11T16:14:08,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:14:08,638 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T16:14:08,650 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T16:14:08,653 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T16:14:08,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T16:14:08,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45121 2024-11-11T16:14:08,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45121 2024-11-11T16:14:08,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45121 2024-11-11T16:14:08,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45121 2024-11-11T16:14:08,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45121 2024-11-11T16:14:08,681 DEBUG [M:0;16b413a53992:33601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;16b413a53992:33601 2024-11-11T16:14:08,682 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/16b413a53992,33601,1731341647570 2024-11-11T16:14:08,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:08,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:08,699 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/16b413a53992,33601,1731341647570 2024-11-11T16:14:08,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T16:14:08,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:08,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:08,722 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T16:14:08,724 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/16b413a53992,33601,1731341647570 from backup master directory 2024-11-11T16:14:08,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/16b413a53992,33601,1731341647570 2024-11-11T16:14:08,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:08,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T16:14:08,728 WARN [master/16b413a53992:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T16:14:08,729 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=16b413a53992,33601,1731341647570 2024-11-11T16:14:08,731 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T16:14:08,733 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T16:14:08,802 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/hbase.id] with ID: bef5b00d-7213-459c-b3e4-26a5a980fa1f 2024-11-11T16:14:08,802 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/.tmp/hbase.id 2024-11-11T16:14:08,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741826_1002 (size=42) 2024-11-11T16:14:08,816 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/.tmp/hbase.id]:[hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/hbase.id] 2024-11-11T16:14:08,892 INFO [master/16b413a53992:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T16:14:08,898 INFO [master/16b413a53992:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T16:14:08,925 INFO [master/16b413a53992:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-11T16:14:08,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:08,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:08,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741827_1003 (size=196) 2024-11-11T16:14:09,386 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:14:09,394 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T16:14:09,418 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:09,426 INFO [master/16b413a53992:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:14:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741828_1004 (size=1189) 2024-11-11T16:14:09,913 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store 2024-11-11T16:14:09,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741829_1005 (size=34) 2024-11-11T16:14:10,346 INFO [master/16b413a53992:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T16:14:10,350 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:10,351 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T16:14:10,351 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:10,352 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:10,353 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T16:14:10,353 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:10,353 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:14:10,355 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731341650351Disabling compacts and flushes for region at 1731341650351Disabling writes for close at 1731341650353 (+2 ms)Writing region close event to WAL at 1731341650353Closed at 1731341650353 2024-11-11T16:14:10,357 WARN [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/.initializing 2024-11-11T16:14:10,357 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/WALs/16b413a53992,33601,1731341647570 2024-11-11T16:14:10,366 INFO [master/16b413a53992:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T16:14:10,383 INFO [master/16b413a53992:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C33601%2C1731341647570, suffix=, logDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/WALs/16b413a53992,33601,1731341647570, archiveDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/oldWALs, maxLogs=10 2024-11-11T16:14:10,408 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/WALs/16b413a53992,33601,1731341647570/16b413a53992%2C33601%2C1731341647570.1731341650388, exclude list is [], retry=0 2024-11-11T16:14:10,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34773,DS-1a41b93d-1191-4ce0-ad40-f93116a298a6,DISK] 2024-11-11T16:14:10,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T16:14:10,471 INFO [master/16b413a53992:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/WALs/16b413a53992,33601,1731341647570/16b413a53992%2C33601%2C1731341647570.1731341650388 2024-11-11T16:14:10,473 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43639:43639)] 2024-11-11T16:14:10,473 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:14:10,474 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:10,479 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,480 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T16:14:10,566 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:10,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:10,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T16:14:10,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:10,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:10,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T16:14:10,582 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:10,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:10,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T16:14:10,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:10,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:10,588 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,594 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,596 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,602 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,602 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,607 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T16:14:10,611 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T16:14:10,615 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:14:10,616 INFO [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61701214, jitterRate=-0.08058026432991028}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T16:14:10,622 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731341650492Initializing all the Stores at 1731341650495 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341650495Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341650496 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341650497 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341650497Cleaning up temporary data from old regions at 1731341650603 (+106 ms)Region opened successfully at 1731341650622 (+19 ms) 2024-11-11T16:14:10,623 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T16:14:10,658 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecf0699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b413a53992/172.17.0.2:0 2024-11-11T16:14:10,691 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T16:14:10,703 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T16:14:10,703 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T16:14:10,707 INFO [master/16b413a53992:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T16:14:10,708 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T16:14:10,713 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T16:14:10,714 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T16:14:10,740 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T16:14:10,749 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T16:14:10,767 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T16:14:10,770 INFO [master/16b413a53992:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T16:14:10,771 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T16:14:10,806 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T16:14:10,809 INFO [master/16b413a53992:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T16:14:10,813 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T16:14:10,860 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T16:14:10,862 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T16:14:10,898 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T16:14:10,918 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T16:14:10,975 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T16:14:11,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T16:14:11,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T16:14:11,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:11,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:11,056 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=16b413a53992,33601,1731341647570, sessionid=0x1002f9f05730000, setting cluster-up flag (Was=false) 2024-11-11T16:14:11,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:11,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:11,430 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T16:14:11,432 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b413a53992,33601,1731341647570 2024-11-11T16:14:11,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:11,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:11,714 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T16:14:11,716 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b413a53992,33601,1731341647570 2024-11-11T16:14:11,723 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T16:14:11,773 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(746): ClusterId : bef5b00d-7213-459c-b3e4-26a5a980fa1f 2024-11-11T16:14:11,776 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T16:14:11,802 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T16:14:11,813 INFO [master/16b413a53992:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T16:14:11,819 INFO [master/16b413a53992:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T16:14:11,824 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 16b413a53992,33601,1731341647570 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T16:14:11,831 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T16:14:11,831 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T16:14:11,832 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:14:11,832 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:14:11,832 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:14:11,832 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/16b413a53992:0, corePoolSize=5, maxPoolSize=5 2024-11-11T16:14:11,832 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/16b413a53992:0, corePoolSize=10, maxPoolSize=10 2024-11-11T16:14:11,833 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:11,833 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/16b413a53992:0, corePoolSize=2, maxPoolSize=2 2024-11-11T16:14:11,833 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:11,837 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731341681837 2024-11-11T16:14:11,838 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T16:14:11,839 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T16:14:11,839 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T16:14:11,839 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T16:14:11,843 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T16:14:11,843 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T16:14:11,843 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T16:14:11,843 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T16:14:11,844 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:11,846 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:11,847 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T16:14:11,847 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T16:14:11,848 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T16:14:11,848 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T16:14:11,850 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T16:14:11,851 INFO [master/16b413a53992:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T16:14:11,853 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.large.0-1731341651852,5,FailOnTimeoutGroup] 2024-11-11T16:14:11,854 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.small.0-1731341651853,5,FailOnTimeoutGroup] 2024-11-11T16:14:11,854 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:11,854 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T16:14:11,856 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:11,856 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:11,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741831_1007 (size=1321) 2024-11-11T16:14:11,892 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T16:14:11,893 DEBUG [RS:0;16b413a53992:45121 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1283c177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b413a53992/172.17.0.2:0 2024-11-11T16:14:11,908 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;16b413a53992:45121 2024-11-11T16:14:11,911 INFO [RS:0;16b413a53992:45121 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T16:14:11,911 INFO [RS:0;16b413a53992:45121 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T16:14:11,911 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T16:14:11,914 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b413a53992,33601,1731341647570 with port=45121, startcode=1731341648557 2024-11-11T16:14:11,925 DEBUG [RS:0;16b413a53992:45121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T16:14:11,959 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47743, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T16:14:11,965 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33601 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b413a53992,45121,1731341648557 2024-11-11T16:14:11,967 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33601 {}] master.ServerManager(517): Registering regionserver=16b413a53992,45121,1731341648557 2024-11-11T16:14:11,983 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:14:11,983 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33383 2024-11-11T16:14:11,984 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T16:14:12,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T16:14:12,084 DEBUG [RS:0;16b413a53992:45121 {}] zookeeper.ZKUtil(111): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b413a53992,45121,1731341648557 2024-11-11T16:14:12,084 WARN [RS:0;16b413a53992:45121 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T16:14:12,084 INFO [RS:0;16b413a53992:45121 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:14:12,084 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557 2024-11-11T16:14:12,087 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b413a53992,45121,1731341648557] 2024-11-11T16:14:12,112 INFO [RS:0;16b413a53992:45121 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T16:14:12,128 INFO [RS:0;16b413a53992:45121 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T16:14:12,133 INFO [RS:0;16b413a53992:45121 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T16:14:12,133 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,134 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T16:14:12,140 INFO [RS:0;16b413a53992:45121 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T16:14:12,141 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,142 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,142 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,142 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,142 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,142 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,142 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b413a53992:0, corePoolSize=2, maxPoolSize=2 2024-11-11T16:14:12,142 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,143 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,143 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,143 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,143 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,143 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b413a53992:0, corePoolSize=1, maxPoolSize=1 2024-11-11T16:14:12,143 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:14:12,143 DEBUG [RS:0;16b413a53992:45121 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0, corePoolSize=3, maxPoolSize=3 2024-11-11T16:14:12,144 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,145 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,145 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,145 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,145 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,145 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,45121,1731341648557-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T16:14:12,165 INFO [RS:0;16b413a53992:45121 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T16:14:12,167 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,45121,1731341648557-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,167 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,167 INFO [RS:0;16b413a53992:45121 {}] regionserver.Replication(171): 16b413a53992,45121,1731341648557 started 2024-11-11T16:14:12,195 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:12,195 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1482): Serving as 16b413a53992,45121,1731341648557, RpcServer on 16b413a53992/172.17.0.2:45121, sessionid=0x1002f9f05730001 2024-11-11T16:14:12,196 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T16:14:12,197 DEBUG [RS:0;16b413a53992:45121 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b413a53992,45121,1731341648557 2024-11-11T16:14:12,197 DEBUG [RS:0;16b413a53992:45121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,45121,1731341648557' 2024-11-11T16:14:12,197 DEBUG [RS:0;16b413a53992:45121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T16:14:12,198 DEBUG [RS:0;16b413a53992:45121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T16:14:12,199 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T16:14:12,199 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T16:14:12,199 DEBUG [RS:0;16b413a53992:45121 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b413a53992,45121,1731341648557 2024-11-11T16:14:12,199 DEBUG [RS:0;16b413a53992:45121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b413a53992,45121,1731341648557' 2024-11-11T16:14:12,200 DEBUG [RS:0;16b413a53992:45121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T16:14:12,200 DEBUG [RS:0;16b413a53992:45121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T16:14:12,201 DEBUG [RS:0;16b413a53992:45121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T16:14:12,201 INFO [RS:0;16b413a53992:45121 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T16:14:12,201 INFO [RS:0;16b413a53992:45121 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T16:14:12,259 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T16:14:12,260 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:14:12,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741832_1008 (size=32) 2024-11-11T16:14:12,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:12,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T16:14:12,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T16:14:12,276 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,277 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,277 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T16:14:12,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T16:14:12,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T16:14:12,283 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T16:14:12,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T16:14:12,286 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T16:14:12,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T16:14:12,289 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740 2024-11-11T16:14:12,289 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740 2024-11-11T16:14:12,292 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T16:14:12,292 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T16:14:12,294 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:14:12,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T16:14:12,301 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:14:12,302 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68910964, jitterRate=0.02685338258743286}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:14:12,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731341652271Initializing all the Stores at 1731341652272 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341652272Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341652273 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341652273Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341652273Cleaning up temporary data from old regions at 1731341652293 (+20 ms)Region opened successfully at 1731341652305 (+12 ms) 2024-11-11T16:14:12,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T16:14:12,305 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T16:14:12,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T16:14:12,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T16:14:12,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T16:14:12,306 INFO [RS:0;16b413a53992:45121 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T16:14:12,307 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T16:14:12,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731341652305Disabling compacts and flushes for region at 1731341652305Disabling writes for close at 1731341652306 (+1 ms)Writing region close event to WAL at 1731341652307 (+1 ms)Closed at 1731341652307 2024-11-11T16:14:12,309 INFO [RS:0;16b413a53992:45121 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C45121%2C1731341648557, suffix=, logDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557, archiveDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/oldWALs, maxLogs=32 2024-11-11T16:14:12,311 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T16:14:12,312 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T16:14:12,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T16:14:12,326 DEBUG [RS:0;16b413a53992:45121 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557/16b413a53992%2C45121%2C1731341648557.1731341652312, exclude list is [], retry=0 2024-11-11T16:14:12,327 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:14:12,330 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T16:14:12,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34773,DS-1a41b93d-1191-4ce0-ad40-f93116a298a6,DISK] 2024-11-11T16:14:12,335 INFO [RS:0;16b413a53992:45121 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557/16b413a53992%2C45121%2C1731341648557.1731341652312 2024-11-11T16:14:12,335 DEBUG [RS:0;16b413a53992:45121 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43639:43639)] 2024-11-11T16:14:12,483 DEBUG [16b413a53992:33601 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T16:14:12,497 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:12,504 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,45121,1731341648557, state=OPENING 2024-11-11T16:14:12,567 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T16:14:12,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:12,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:14:12,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:14:12,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:14:12,655 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T16:14:12,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:14:12,835 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T16:14:12,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52509, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T16:14:12,851 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T16:14:12,851 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T16:14:12,852 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T16:14:12,855 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b413a53992%2C45121%2C1731341648557.meta, suffix=.meta, logDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557, archiveDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/oldWALs, maxLogs=32 2024-11-11T16:14:12,872 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557/16b413a53992%2C45121%2C1731341648557.meta.1731341652857.meta, exclude list is [], retry=0 2024-11-11T16:14:12,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34773,DS-1a41b93d-1191-4ce0-ad40-f93116a298a6,DISK] 2024-11-11T16:14:12,880 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557/16b413a53992%2C45121%2C1731341648557.meta.1731341652857.meta 2024-11-11T16:14:12,881 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43639:43639)] 2024-11-11T16:14:12,881 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:14:12,883 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T16:14:12,886 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T16:14:12,891 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T16:14:12,895 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T16:14:12,896 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:12,897 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T16:14:12,897 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T16:14:12,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T16:14:12,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T16:14:12,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T16:14:12,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T16:14:12,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T16:14:12,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T16:14:12,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T16:14:12,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T16:14:12,909 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:12,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T16:14:12,910 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T16:14:12,911 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740 2024-11-11T16:14:12,914 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740 2024-11-11T16:14:12,916 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T16:14:12,916 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T16:14:12,918 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:14:12,920 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T16:14:12,922 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67728260, jitterRate=0.009229719638824463}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:14:12,922 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T16:14:12,924 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731341652897Writing region info on filesystem at 1731341652897Initializing all the Stores at 1731341652899 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341652899Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341652900 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341652900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731341652900Cleaning up temporary data from old regions at 1731341652917 (+17 ms)Running coprocessor post-open hooks at 1731341652922 (+5 ms)Region opened successfully at 1731341652924 (+2 ms) 2024-11-11T16:14:12,933 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731341652825 2024-11-11T16:14:12,944 DEBUG [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T16:14:12,945 INFO [RS_OPEN_META-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T16:14:12,947 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:12,949 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b413a53992,45121,1731341648557, state=OPEN 2024-11-11T16:14:13,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:14:13,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T16:14:13,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:14:13,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T16:14:13,158 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:13,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T16:14:13,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b413a53992,45121,1731341648557 in 501 msec 2024-11-11T16:14:13,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T16:14:13,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 849 msec 2024-11-11T16:14:13,173 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T16:14:13,174 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T16:14:13,196 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:13,198 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:13,220 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:13,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33033, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:13,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4880 sec 2024-11-11T16:14:13,242 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731341653242, completionTime=-1 2024-11-11T16:14:13,245 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T16:14:13,245 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T16:14:13,296 INFO [master/16b413a53992:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-11T16:14:13,296 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731341713296 2024-11-11T16:14:13,296 INFO [master/16b413a53992:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731341773296 2024-11-11T16:14:13,296 INFO [master/16b413a53992:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 50 msec 2024-11-11T16:14:13,298 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33601,1731341647570-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:13,299 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33601,1731341647570-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:13,299 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33601,1731341647570-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:13,300 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-16b413a53992:33601, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:13,301 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:13,303 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:13,308 DEBUG [master/16b413a53992:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T16:14:13,339 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.610sec 2024-11-11T16:14:13,341 INFO [master/16b413a53992:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T16:14:13,342 INFO [master/16b413a53992:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T16:14:13,343 INFO [master/16b413a53992:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T16:14:13,344 INFO [master/16b413a53992:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T16:14:13,344 INFO [master/16b413a53992:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T16:14:13,344 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33601,1731341647570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T16:14:13,345 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33601,1731341647570-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T16:14:13,353 DEBUG [master/16b413a53992:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T16:14:13,354 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T16:14:13,354 INFO [master/16b413a53992:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b413a53992,33601,1731341647570-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T16:14:13,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dfd3b84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:13,403 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T16:14:13,403 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T16:14:13,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:13,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:13,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531238f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:13,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:13,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346890e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:13,503 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:13,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:13,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=16b413a53992,33601,1731341647570 2024-11-11T16:14:13,546 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=544, ProcessCount=11, AvailableMemoryMB=2614 2024-11-11T16:14:13,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:14:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,33601,1731341647570 2024-11-11T16:14:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@680215db 2024-11-11T16:14:13,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:14:13,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:14:13,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:14:13,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:14:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:13,582 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T16:14:13,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T16:14:13,585 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:13,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T16:14:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:14:13,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741835_1011 (size=1061) 2024-11-11T16:14:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:14:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:14:14,002 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:14:14,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741836_1012 (size=53) 2024-11-11T16:14:14,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:14,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing af9dedd7afe35ed4a9eb53f0210465b1, disabling compactions & flushes 2024-11-11T16:14:14,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. after waiting 0 ms 2024-11-11T16:14:14,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,014 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for af9dedd7afe35ed4a9eb53f0210465b1: Waiting for close lock at 1731341654013Disabling compacts and flushes for region at 1731341654013Disabling writes for close at 1731341654014 (+1 ms)Writing region close event to WAL at 1731341654014Closed at 1731341654014 2024-11-11T16:14:14,017 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T16:14:14,022 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731341654017"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341654017"}]},"ts":"1731341654017"} 2024-11-11T16:14:14,028 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T16:14:14,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T16:14:14,033 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341654030"}]},"ts":"1731341654030"} 2024-11-11T16:14:14,038 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T16:14:14,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, ASSIGN}] 2024-11-11T16:14:14,043 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, ASSIGN 2024-11-11T16:14:14,044 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, ASSIGN; state=OFFLINE, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=false 2024-11-11T16:14:14,196 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=af9dedd7afe35ed4a9eb53f0210465b1, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:14,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, ASSIGN because future has completed 2024-11-11T16:14:14,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:14:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:14:14,364 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,364 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:14:14,365 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,365 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:14,365 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,365 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,367 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,371 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:14,371 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af9dedd7afe35ed4a9eb53f0210465b1 columnFamilyName A 2024-11-11T16:14:14,371 DEBUG [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:14,372 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.HStore(327): Store=af9dedd7afe35ed4a9eb53f0210465b1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:14,373 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,374 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:14,375 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af9dedd7afe35ed4a9eb53f0210465b1 columnFamilyName B 2024-11-11T16:14:14,375 DEBUG [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:14,376 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.HStore(327): Store=af9dedd7afe35ed4a9eb53f0210465b1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:14,376 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,378 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:14,379 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af9dedd7afe35ed4a9eb53f0210465b1 columnFamilyName C 2024-11-11T16:14:14,379 DEBUG [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:14,380 INFO [StoreOpener-af9dedd7afe35ed4a9eb53f0210465b1-1 {}] regionserver.HStore(327): Store=af9dedd7afe35ed4a9eb53f0210465b1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:14,380 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,381 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,382 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,383 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,385 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,385 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,386 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:14:14,389 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,393 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:14:14,393 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened af9dedd7afe35ed4a9eb53f0210465b1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61350488, jitterRate=-0.08580648899078369}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:14:14,394 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,395 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for af9dedd7afe35ed4a9eb53f0210465b1: Running coprocessor pre-open hook at 1731341654365Writing region info on filesystem at 1731341654365Initializing all the Stores at 1731341654367 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341654367Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341654367Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341654367Cleaning up temporary data from old regions at 1731341654385 (+18 ms)Running coprocessor post-open hooks at 1731341654394 (+9 ms)Region opened successfully at 1731341654394 2024-11-11T16:14:14,397 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., pid=6, masterSystemTime=1731341654357 2024-11-11T16:14:14,400 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,400 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:14,402 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=af9dedd7afe35ed4a9eb53f0210465b1, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:14,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:14:14,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T16:14:14,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 in 205 msec 2024-11-11T16:14:14,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T16:14:14,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, ASSIGN in 372 msec 2024-11-11T16:14:14,418 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T16:14:14,419 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341654418"}]},"ts":"1731341654418"} 2024-11-11T16:14:14,422 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T16:14:14,424 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T16:14:14,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 849 msec 2024-11-11T16:14:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T16:14:14,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:14,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ccbf31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,736 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328e9320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,740 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec4a323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,749 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:14:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,33601,1731341647570 2024-11-11T16:14:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@30e64719 2024-11-11T16:14:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:14:14,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:14:14,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1417864b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,764 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e9c175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,775 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219bef75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,783 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,786 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2125c79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,794 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e412ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0c2a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,798 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c2308e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,800 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee049f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72aafc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,804 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ad14c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494cfe15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,810 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc5ceca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46783332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed37001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e877dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,818 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35351cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea06d55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:14,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:14,821 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e873e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:14,823 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21616163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:14,828 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,828 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,829 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,829 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,830 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,831 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,831 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,831 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,833 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T16:14:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,845 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T16:14:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:14,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:14,850 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:14,853 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:14:14,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:14,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:14:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:14,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:14,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:14,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:14,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:14,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T16:14:15,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T16:14:15,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/feaaf95e76024f96b22f4ee5d2d2ed49 is 50, key is test_row_0/A:col10/1731341654892/Put/seqid=0 2024-11-11T16:14:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341715089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741837_1013 (size=21365) 2024-11-11T16:14:15,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/feaaf95e76024f96b22f4ee5d2d2ed49 2024-11-11T16:14:15,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341715107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341715111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341715090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341715119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T16:14:15,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T16:14:15,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:15,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b3a5e1867c3d47f092948029771b8d44 is 50, key is test_row_0/B:col10/1731341654892/Put/seqid=0 2024-11-11T16:14:15,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341715281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341715282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341715282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341715283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341715283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741838_1014 (size=12001) 2024-11-11T16:14:15,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b3a5e1867c3d47f092948029771b8d44 2024-11-11T16:14:15,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/ea00cea79f554709b306b02f3b24af46 is 50, key is test_row_0/C:col10/1731341654892/Put/seqid=0 2024-11-11T16:14:15,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T16:14:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741839_1015 (size=12001) 2024-11-11T16:14:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T16:14:15,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341715498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341715502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341715509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341715509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341715501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T16:14:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:15,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T16:14:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:15,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341715818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341715827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341715817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341715828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341715837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:15,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:15,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/ea00cea79f554709b306b02f3b24af46 2024-11-11T16:14:15,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/feaaf95e76024f96b22f4ee5d2d2ed49 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/feaaf95e76024f96b22f4ee5d2d2ed49 2024-11-11T16:14:15,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/feaaf95e76024f96b22f4ee5d2d2ed49, entries=350, sequenceid=14, filesize=20.9 K 2024-11-11T16:14:15,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b3a5e1867c3d47f092948029771b8d44 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b3a5e1867c3d47f092948029771b8d44 2024-11-11T16:14:15,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T16:14:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:15,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:15,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b3a5e1867c3d47f092948029771b8d44, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T16:14:15,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/ea00cea79f554709b306b02f3b24af46 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ea00cea79f554709b306b02f3b24af46 2024-11-11T16:14:15,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ea00cea79f554709b306b02f3b24af46, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T16:14:15,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for af9dedd7afe35ed4a9eb53f0210465b1 in 1034ms, sequenceid=14, compaction requested=false 2024-11-11T16:14:15,971 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T16:14:15,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T16:14:16,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T16:14:16,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:16,057 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:16,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b9be62f619b14c3685d0eb63f06ef734 is 50, key is test_row_0/A:col10/1731341655079/Put/seqid=0 2024-11-11T16:14:16,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741840_1016 (size=12001) 2024-11-11T16:14:16,135 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b9be62f619b14c3685d0eb63f06ef734 2024-11-11T16:14:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/29818c46503c421fbc8af32d2932731a is 50, key is test_row_0/B:col10/1731341655079/Put/seqid=0 2024-11-11T16:14:16,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741841_1017 (size=12001) 2024-11-11T16:14:16,263 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/29818c46503c421fbc8af32d2932731a 2024-11-11T16:14:16,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/1eba8ecc14f04a1db8f7ba3e2afe941b is 50, key is test_row_0/C:col10/1731341655079/Put/seqid=0 2024-11-11T16:14:16,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741842_1018 (size=12001) 2024-11-11T16:14:16,348 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/1eba8ecc14f04a1db8f7ba3e2afe941b 2024-11-11T16:14:16,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b9be62f619b14c3685d0eb63f06ef734 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b9be62f619b14c3685d0eb63f06ef734 2024-11-11T16:14:16,390 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b9be62f619b14c3685d0eb63f06ef734, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T16:14:16,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/29818c46503c421fbc8af32d2932731a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/29818c46503c421fbc8af32d2932731a 2024-11-11T16:14:16,415 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/29818c46503c421fbc8af32d2932731a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T16:14:16,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341716392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341716393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341716397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/1eba8ecc14f04a1db8f7ba3e2afe941b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/1eba8ecc14f04a1db8f7ba3e2afe941b 2024-11-11T16:14:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341716415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341716419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,442 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/1eba8ecc14f04a1db8f7ba3e2afe941b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T16:14:16,446 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for af9dedd7afe35ed4a9eb53f0210465b1 in 389ms, sequenceid=33, compaction requested=false 2024-11-11T16:14:16,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:16,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:16,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T16:14:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T16:14:16,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T16:14:16,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5940 sec 2024-11-11T16:14:16,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6190 sec 2024-11-11T16:14:16,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:16,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/c6d71d062fce4477a7c236f3e2c5665b is 50, key is test_row_0/A:col10/1731341656412/Put/seqid=0 2024-11-11T16:14:16,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741843_1019 (size=14341) 2024-11-11T16:14:16,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341716620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341716636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341716641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341716644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341716650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341716759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341716766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341716767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341716767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341716777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341716978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341716988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T16:14:16,998 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:16,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341716992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341716993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341716997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T16:14:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T16:14:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,005 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:17,007 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:17,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:17,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/c6d71d062fce4477a7c236f3e2c5665b 2024-11-11T16:14:17,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/9dc959f2f6074b36b123f21bafed94af is 50, key is test_row_0/B:col10/1731341656412/Put/seqid=0 2024-11-11T16:14:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T16:14:17,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741844_1020 (size=12001) 2024-11-11T16:14:17,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T16:14:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/9dc959f2f6074b36b123f21bafed94af 2024-11-11T16:14:17,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/0614c82fcf674d5b933ac061added1ec is 50, key is test_row_0/C:col10/1731341656412/Put/seqid=0 2024-11-11T16:14:17,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741845_1021 (size=12001) 2024-11-11T16:14:17,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341717298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341717308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341717308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T16:14:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T16:14:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341717316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T16:14:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T16:14:17,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T16:14:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:17,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/0614c82fcf674d5b933ac061added1ec 2024-11-11T16:14:17,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/c6d71d062fce4477a7c236f3e2c5665b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/c6d71d062fce4477a7c236f3e2c5665b 2024-11-11T16:14:17,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/c6d71d062fce4477a7c236f3e2c5665b, entries=200, sequenceid=48, filesize=14.0 K 2024-11-11T16:14:17,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/9dc959f2f6074b36b123f21bafed94af as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/9dc959f2f6074b36b123f21bafed94af 2024-11-11T16:14:17,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/9dc959f2f6074b36b123f21bafed94af, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T16:14:17,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/0614c82fcf674d5b933ac061added1ec as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/0614c82fcf674d5b933ac061added1ec 2024-11-11T16:14:17,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/0614c82fcf674d5b933ac061added1ec, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T16:14:17,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for af9dedd7afe35ed4a9eb53f0210465b1 in 1216ms, sequenceid=48, compaction requested=true 2024-11-11T16:14:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:17,779 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:17,785 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:17,786 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:17,788 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:17,789 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,790 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b3a5e1867c3d47f092948029771b8d44, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/29818c46503c421fbc8af32d2932731a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/9dc959f2f6074b36b123f21bafed94af] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.2 K 2024-11-11T16:14:17,792 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a5e1867c3d47f092948029771b8d44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731341654892 2024-11-11T16:14:17,794 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 29818c46503c421fbc8af32d2932731a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731341655053 2024-11-11T16:14:17,795 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dc959f2f6074b36b123f21bafed94af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341656392 2024-11-11T16:14:17,797 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47707 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:17,797 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:17,797 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,797 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/feaaf95e76024f96b22f4ee5d2d2ed49, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b9be62f619b14c3685d0eb63f06ef734, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/c6d71d062fce4477a7c236f3e2c5665b] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=46.6 K 2024-11-11T16:14:17,798 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting feaaf95e76024f96b22f4ee5d2d2ed49, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731341654877 2024-11-11T16:14:17,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T16:14:17,799 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9be62f619b14c3685d0eb63f06ef734, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731341655053 2024-11-11T16:14:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,800 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T16:14:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:17,802 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d71d062fce4477a7c236f3e2c5665b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341656392 2024-11-11T16:14:17,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f4a8b1ad6ae540dc8c2917535cfc9096 is 50, key is test_row_0/A:col10/1731341656632/Put/seqid=0 2024-11-11T16:14:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:17,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741846_1022 (size=12001) 2024-11-11T16:14:17,862 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f4a8b1ad6ae540dc8c2917535cfc9096 2024-11-11T16:14:17,877 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#11 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:17,878 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/5b912354d7f1497db6810e1104a1c270 is 50, key is test_row_0/B:col10/1731341656412/Put/seqid=0 2024-11-11T16:14:17,879 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#10 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:17,880 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/48489375dec74b01ad7ea92b8172776f is 50, key is test_row_0/A:col10/1731341656412/Put/seqid=0 2024-11-11T16:14:17,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341717865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341717874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/924f4c8382ab4f08844b7ac96e93d716 is 50, key is test_row_0/B:col10/1731341656632/Put/seqid=0 2024-11-11T16:14:17,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341717879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341717884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341717888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:17,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:17,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741847_1023 (size=12104) 2024-11-11T16:14:17,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741848_1024 (size=12104) 2024-11-11T16:14:17,932 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/5b912354d7f1497db6810e1104a1c270 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/5b912354d7f1497db6810e1104a1c270 2024-11-11T16:14:17,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741849_1025 (size=12001) 2024-11-11T16:14:17,958 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into 5b912354d7f1497db6810e1104a1c270(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:17,959 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:17,962 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341657775; duration=0sec 2024-11-11T16:14:17,963 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:17,964 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:17,964 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:17,978 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:17,978 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:17,979 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:17,979 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ea00cea79f554709b306b02f3b24af46, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/1eba8ecc14f04a1db8f7ba3e2afe941b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/0614c82fcf674d5b933ac061added1ec] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.2 K 2024-11-11T16:14:17,980 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ea00cea79f554709b306b02f3b24af46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731341654892 2024-11-11T16:14:17,983 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eba8ecc14f04a1db8f7ba3e2afe941b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731341655053 2024-11-11T16:14:17,989 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0614c82fcf674d5b933ac061added1ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341656392 2024-11-11T16:14:18,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341717997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341717998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341718016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341718017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,028 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#13 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:18,029 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/57f3a4e08c5241f59e0f80b593fe94a8 is 50, key is test_row_0/C:col10/1731341656412/Put/seqid=0 2024-11-11T16:14:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341718017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741850_1026 (size=12104) 2024-11-11T16:14:18,084 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/57f3a4e08c5241f59e0f80b593fe94a8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/57f3a4e08c5241f59e0f80b593fe94a8 2024-11-11T16:14:18,100 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into 57f3a4e08c5241f59e0f80b593fe94a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:18,100 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:18,100 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341657776; duration=0sec 2024-11-11T16:14:18,100 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:18,100 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T16:14:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341718217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341718217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341718226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341718236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341718246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,335 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/924f4c8382ab4f08844b7ac96e93d716 2024-11-11T16:14:18,349 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/48489375dec74b01ad7ea92b8172776f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/48489375dec74b01ad7ea92b8172776f 2024-11-11T16:14:18,367 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 48489375dec74b01ad7ea92b8172776f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:18,367 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:18,368 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341657757; duration=0sec 2024-11-11T16:14:18,368 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:18,368 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:18,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/c7f1cb2a988b4eee86b3e1f3764084e6 is 50, key is test_row_0/C:col10/1731341656632/Put/seqid=0 2024-11-11T16:14:18,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741851_1027 (size=12001) 2024-11-11T16:14:18,409 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T16:14:18,494 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T16:14:18,499 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T16:14:18,500 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T16:14:18,502 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T16:14:18,502 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T16:14:18,502 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T16:14:18,502 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T16:14:18,503 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T16:14:18,503 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T16:14:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341718536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341718536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341718546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341718546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341718566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:18,796 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/c7f1cb2a988b4eee86b3e1f3764084e6 2024-11-11T16:14:18,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f4a8b1ad6ae540dc8c2917535cfc9096 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f4a8b1ad6ae540dc8c2917535cfc9096 2024-11-11T16:14:18,835 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f4a8b1ad6ae540dc8c2917535cfc9096, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T16:14:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/924f4c8382ab4f08844b7ac96e93d716 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/924f4c8382ab4f08844b7ac96e93d716 2024-11-11T16:14:18,857 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/924f4c8382ab4f08844b7ac96e93d716, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T16:14:18,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/c7f1cb2a988b4eee86b3e1f3764084e6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c7f1cb2a988b4eee86b3e1f3764084e6 2024-11-11T16:14:18,879 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c7f1cb2a988b4eee86b3e1f3764084e6, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T16:14:18,882 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for af9dedd7afe35ed4a9eb53f0210465b1 in 1081ms, sequenceid=65, compaction requested=false 2024-11-11T16:14:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T16:14:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T16:14:18,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T16:14:18,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-11T16:14:18,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8920 sec 2024-11-11T16:14:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:19,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T16:14:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:19,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/e938230e4a0741ceadf7a5693bbc8dcd is 50, key is test_row_0/A:col10/1731341659050/Put/seqid=0 2024-11-11T16:14:19,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341719088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341719088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341719091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341719092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341719094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741852_1028 (size=12001) 2024-11-11T16:14:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T16:14:19,157 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:19,158 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T16:14:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T16:14:19,164 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:19,166 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:19,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:19,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341719207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341719207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341719208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341719208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341719208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T16:14:19,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T16:14:19,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:19,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341719415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341719416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341719416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341719416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341719417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T16:14:19,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:19,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T16:14:19,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/e938230e4a0741ceadf7a5693bbc8dcd 2024-11-11T16:14:19,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b975ad3b681a42b3b3e13f18a8cc2687 is 50, key is test_row_0/B:col10/1731341659050/Put/seqid=0 2024-11-11T16:14:19,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741853_1029 (size=12001) 2024-11-11T16:14:19,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T16:14:19,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:19,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341719727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341719728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341719727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341719728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341719728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:19,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:19,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T16:14:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T16:14:19,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T16:14:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:19,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:19,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b975ad3b681a42b3b3e13f18a8cc2687 2024-11-11T16:14:19,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/c628215bc339485bab467eafb0832ab7 is 50, key is test_row_0/C:col10/1731341659050/Put/seqid=0 2024-11-11T16:14:20,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741854_1030 (size=12001) 2024-11-11T16:14:20,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/c628215bc339485bab467eafb0832ab7 2024-11-11T16:14:20,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/e938230e4a0741ceadf7a5693bbc8dcd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e938230e4a0741ceadf7a5693bbc8dcd 2024-11-11T16:14:20,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e938230e4a0741ceadf7a5693bbc8dcd, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T16:14:20,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b975ad3b681a42b3b3e13f18a8cc2687 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b975ad3b681a42b3b3e13f18a8cc2687 2024-11-11T16:14:20,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b975ad3b681a42b3b3e13f18a8cc2687, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T16:14:20,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/c628215bc339485bab467eafb0832ab7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c628215bc339485bab467eafb0832ab7 2024-11-11T16:14:20,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c628215bc339485bab467eafb0832ab7, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T16:14:20,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for af9dedd7afe35ed4a9eb53f0210465b1 in 1047ms, sequenceid=84, compaction requested=true 2024-11-11T16:14:20,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:20,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:20,103 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:20,103 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:20,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T16:14:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:20,105 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:20,105 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:20,105 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:20,105 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T16:14:20,105 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/48489375dec74b01ad7ea92b8172776f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f4a8b1ad6ae540dc8c2917535cfc9096, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e938230e4a0741ceadf7a5693bbc8dcd] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.3 K 2024-11-11T16:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:20,106 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48489375dec74b01ad7ea92b8172776f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341656392 2024-11-11T16:14:20,106 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:20,106 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:20,107 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:20,107 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/5b912354d7f1497db6810e1104a1c270, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/924f4c8382ab4f08844b7ac96e93d716, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b975ad3b681a42b3b3e13f18a8cc2687] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.3 K 2024-11-11T16:14:20,107 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4a8b1ad6ae540dc8c2917535cfc9096, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731341656589 2024-11-11T16:14:20,108 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e938230e4a0741ceadf7a5693bbc8dcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341657873 2024-11-11T16:14:20,108 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b912354d7f1497db6810e1104a1c270, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341656392 2024-11-11T16:14:20,109 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 924f4c8382ab4f08844b7ac96e93d716, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731341656589 2024-11-11T16:14:20,110 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b975ad3b681a42b3b3e13f18a8cc2687, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341657873 2024-11-11T16:14:20,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/a903e129ce1a4fb6bab46fe3545deec1 is 50, key is test_row_0/A:col10/1731341659090/Put/seqid=0 2024-11-11T16:14:20,132 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:20,134 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b7f51055f172450f964b7b84faf5dbbb is 50, key is test_row_0/B:col10/1731341659050/Put/seqid=0 2024-11-11T16:14:20,141 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:20,142 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/1e996c3eac184fda88ec0a57cab34fda is 50, key is test_row_0/A:col10/1731341659050/Put/seqid=0 2024-11-11T16:14:20,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741855_1031 (size=12001) 2024-11-11T16:14:20,158 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/a903e129ce1a4fb6bab46fe3545deec1 2024-11-11T16:14:20,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741856_1032 (size=12207) 2024-11-11T16:14:20,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741857_1033 (size=12207) 2024-11-11T16:14:20,183 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/b7f51055f172450f964b7b84faf5dbbb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b7f51055f172450f964b7b84faf5dbbb 2024-11-11T16:14:20,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d3ac281fda314f8b8c03b15939357f3d is 50, key is test_row_0/B:col10/1731341659090/Put/seqid=0 2024-11-11T16:14:20,189 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/1e996c3eac184fda88ec0a57cab34fda as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1e996c3eac184fda88ec0a57cab34fda 2024-11-11T16:14:20,199 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into b7f51055f172450f964b7b84faf5dbbb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:20,200 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:20,200 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341660103; duration=0sec 2024-11-11T16:14:20,200 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:20,200 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:20,200 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:20,228 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 1e996c3eac184fda88ec0a57cab34fda(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:20,228 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:20,229 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341660102; duration=0sec 2024-11-11T16:14:20,229 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:20,229 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:20,235 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:20,235 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:20,235 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:20,236 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/57f3a4e08c5241f59e0f80b593fe94a8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c7f1cb2a988b4eee86b3e1f3764084e6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c628215bc339485bab467eafb0832ab7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.3 K 2024-11-11T16:14:20,236 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f3a4e08c5241f59e0f80b593fe94a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341656392 2024-11-11T16:14:20,237 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f1cb2a988b4eee86b3e1f3764084e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731341656589 2024-11-11T16:14:20,238 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c628215bc339485bab467eafb0832ab7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341657873 2024-11-11T16:14:20,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:20,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741858_1034 (size=12001) 2024-11-11T16:14:20,269 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d3ac281fda314f8b8c03b15939357f3d 2024-11-11T16:14:20,277 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:20,279 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a80a4fa2457440d19f78957eb9fe6ef0 is 50, key is test_row_0/C:col10/1731341659050/Put/seqid=0 2024-11-11T16:14:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/5500718ed30f40f99aa29c21f64c9b4d is 50, key is test_row_0/C:col10/1731341659090/Put/seqid=0 2024-11-11T16:14:20,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341720280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341720281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341720284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341720292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341720289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T16:14:20,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741859_1035 (size=12207) 2024-11-11T16:14:20,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741860_1036 (size=12001) 2024-11-11T16:14:20,320 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/5500718ed30f40f99aa29c21f64c9b4d 2024-11-11T16:14:20,331 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a80a4fa2457440d19f78957eb9fe6ef0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a80a4fa2457440d19f78957eb9fe6ef0 2024-11-11T16:14:20,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/a903e129ce1a4fb6bab46fe3545deec1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a903e129ce1a4fb6bab46fe3545deec1 2024-11-11T16:14:20,343 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a903e129ce1a4fb6bab46fe3545deec1, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T16:14:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d3ac281fda314f8b8c03b15939357f3d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d3ac281fda314f8b8c03b15939357f3d 2024-11-11T16:14:20,347 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into a80a4fa2457440d19f78957eb9fe6ef0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:20,348 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:20,348 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341660103; duration=0sec 2024-11-11T16:14:20,348 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:20,348 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:20,359 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d3ac281fda314f8b8c03b15939357f3d, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T16:14:20,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/5500718ed30f40f99aa29c21f64c9b4d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5500718ed30f40f99aa29c21f64c9b4d 2024-11-11T16:14:20,375 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5500718ed30f40f99aa29c21f64c9b4d, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T16:14:20,379 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for af9dedd7afe35ed4a9eb53f0210465b1 in 274ms, sequenceid=100, compaction requested=false 2024-11-11T16:14:20,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T16:14:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T16:14:20,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T16:14:20,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2190 sec 2024-11-11T16:14:20,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2390 sec 2024-11-11T16:14:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:20,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:14:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:20,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:20,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:20,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/d90539be4a064526a99a1c036a62c0de is 50, key is test_row_0/A:col10/1731341660410/Put/seqid=0 2024-11-11T16:14:20,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341720431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341720438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341720438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341720441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341720442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741861_1037 (size=12001) 2024-11-11T16:14:20,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/d90539be4a064526a99a1c036a62c0de 2024-11-11T16:14:20,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/80d8c291021a4850ad582cf01891e377 is 50, key is test_row_0/B:col10/1731341660410/Put/seqid=0 2024-11-11T16:14:20,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741862_1038 (size=12001) 2024-11-11T16:14:20,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/80d8c291021a4850ad582cf01891e377 2024-11-11T16:14:20,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/b053b0abba2d44fca0033ab6dca79a8d is 50, key is test_row_0/C:col10/1731341660410/Put/seqid=0 2024-11-11T16:14:20,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741863_1039 (size=12001) 2024-11-11T16:14:20,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341720546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341720557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341720557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341720557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341720557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341720756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341720766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341720766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341720767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341720767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:20,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:20,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:20,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/b053b0abba2d44fca0033ab6dca79a8d 2024-11-11T16:14:20,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/d90539be4a064526a99a1c036a62c0de as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/d90539be4a064526a99a1c036a62c0de 2024-11-11T16:14:20,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/d90539be4a064526a99a1c036a62c0de, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T16:14:20,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/80d8c291021a4850ad582cf01891e377 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/80d8c291021a4850ad582cf01891e377 2024-11-11T16:14:20,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/80d8c291021a4850ad582cf01891e377, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T16:14:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/b053b0abba2d44fca0033ab6dca79a8d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/b053b0abba2d44fca0033ab6dca79a8d 2024-11-11T16:14:20,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/b053b0abba2d44fca0033ab6dca79a8d, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T16:14:20,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for af9dedd7afe35ed4a9eb53f0210465b1 in 579ms, sequenceid=122, compaction requested=true 2024-11-11T16:14:20,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:20,992 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:20,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:20,993 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:20,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:20,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:20,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:20,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:20,997 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:20,997 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:20,997 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:20,997 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1e996c3eac184fda88ec0a57cab34fda, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a903e129ce1a4fb6bab46fe3545deec1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/d90539be4a064526a99a1c036a62c0de] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.4 K 2024-11-11T16:14:20,998 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:20,998 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e996c3eac184fda88ec0a57cab34fda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341657873 2024-11-11T16:14:20,998 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:20,998 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:20,998 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b7f51055f172450f964b7b84faf5dbbb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d3ac281fda314f8b8c03b15939357f3d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/80d8c291021a4850ad582cf01891e377] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.4 K 2024-11-11T16:14:20,999 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a903e129ce1a4fb6bab46fe3545deec1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731341659084 2024-11-11T16:14:20,999 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90539be4a064526a99a1c036a62c0de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731341660406 2024-11-11T16:14:20,999 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b7f51055f172450f964b7b84faf5dbbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341657873 2024-11-11T16:14:21,001 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d3ac281fda314f8b8c03b15939357f3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731341659084 2024-11-11T16:14:21,001 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d8c291021a4850ad582cf01891e377, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731341660406 2024-11-11T16:14:21,023 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:21,024 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/4d7bed22c2b0435eb9388e681a80513a is 50, key is test_row_0/A:col10/1731341660410/Put/seqid=0 2024-11-11T16:14:21,034 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:21,035 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f05507cdb085418f8cb2d7ef6c6d33fe is 50, key is test_row_0/B:col10/1731341660410/Put/seqid=0 2024-11-11T16:14:21,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741864_1040 (size=12309) 2024-11-11T16:14:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741865_1041 (size=12309) 2024-11-11T16:14:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:21,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:14:21,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:21,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:21,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:21,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:21,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:21,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:21,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/cec164d9003142498f850f360bd68b34 is 50, key is test_row_0/A:col10/1731341660438/Put/seqid=0 2024-11-11T16:14:21,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741866_1042 (size=14541) 2024-11-11T16:14:21,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341721110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341721111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341721125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341721126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341721125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341721236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341721236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341721246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341721247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341721247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T16:14:21,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:21,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T16:14:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T16:14:21,319 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:21,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:21,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T16:14:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341721446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341721446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341721456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341721457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341721457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,479 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/4d7bed22c2b0435eb9388e681a80513a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/4d7bed22c2b0435eb9388e681a80513a 2024-11-11T16:14:21,482 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f05507cdb085418f8cb2d7ef6c6d33fe as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f05507cdb085418f8cb2d7ef6c6d33fe 2024-11-11T16:14:21,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:21,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:21,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,495 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into f05507cdb085418f8cb2d7ef6c6d33fe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:21,495 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 4d7bed22c2b0435eb9388e681a80513a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:21,495 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:21,495 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:21,495 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341660993; duration=0sec 2024-11-11T16:14:21,495 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341660992; duration=0sec 2024-11-11T16:14:21,495 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:21,495 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:21,495 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:21,496 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:21,496 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:21,498 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:21,498 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:21,498 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,499 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a80a4fa2457440d19f78957eb9fe6ef0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5500718ed30f40f99aa29c21f64c9b4d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/b053b0abba2d44fca0033ab6dca79a8d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=35.4 K 2024-11-11T16:14:21,499 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a80a4fa2457440d19f78957eb9fe6ef0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341657873 2024-11-11T16:14:21,500 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5500718ed30f40f99aa29c21f64c9b4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731341659084 2024-11-11T16:14:21,501 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b053b0abba2d44fca0033ab6dca79a8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731341660406 2024-11-11T16:14:21,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/cec164d9003142498f850f360bd68b34 2024-11-11T16:14:21,522 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:21,523 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/9871d226eced4f4293a1d8a58564b368 is 50, key is test_row_0/C:col10/1731341660410/Put/seqid=0 2024-11-11T16:14:21,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/3d85ca404c2b4580986d9581334450ca is 50, key is test_row_0/B:col10/1731341660438/Put/seqid=0 2024-11-11T16:14:21,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741867_1043 (size=12309) 2024-11-11T16:14:21,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741868_1044 (size=12151) 2024-11-11T16:14:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T16:14:21,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:21,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:21,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341721756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341721758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341721777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341721777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341721777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:21,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T16:14:21,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:21,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:21,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:21,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:21,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/3d85ca404c2b4580986d9581334450ca 2024-11-11T16:14:21,982 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/9871d226eced4f4293a1d8a58564b368 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/9871d226eced4f4293a1d8a58564b368 2024-11-11T16:14:21,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/dbc051e1978142bbb258b2d941f6226d is 50, key is test_row_0/C:col10/1731341660438/Put/seqid=0 2024-11-11T16:14:22,015 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into 9871d226eced4f4293a1d8a58564b368(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:22,015 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:22,016 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341660995; duration=0sec 2024-11-11T16:14:22,016 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:22,016 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:22,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741869_1045 (size=12151) 2024-11-11T16:14:22,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:22,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341722266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:22,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:22,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341722275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:22,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341722286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341722287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:22,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341722294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:22,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:22,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:22,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:22,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:22,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/dbc051e1978142bbb258b2d941f6226d 2024-11-11T16:14:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:22,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/cec164d9003142498f850f360bd68b34 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cec164d9003142498f850f360bd68b34 2024-11-11T16:14:22,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cec164d9003142498f850f360bd68b34, entries=200, sequenceid=137, filesize=14.2 K 2024-11-11T16:14:22,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/3d85ca404c2b4580986d9581334450ca as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/3d85ca404c2b4580986d9581334450ca 2024-11-11T16:14:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T16:14:22,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/3d85ca404c2b4580986d9581334450ca, entries=150, sequenceid=137, filesize=11.9 K 2024-11-11T16:14:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/dbc051e1978142bbb258b2d941f6226d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/dbc051e1978142bbb258b2d941f6226d 2024-11-11T16:14:22,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/dbc051e1978142bbb258b2d941f6226d, entries=150, sequenceid=137, filesize=11.9 K 2024-11-11T16:14:22,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for af9dedd7afe35ed4a9eb53f0210465b1 in 1400ms, sequenceid=137, compaction requested=false 2024-11-11T16:14:22,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:22,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T16:14:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:22,586 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T16:14:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:22,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/6c1072d91e184161867a09c540646cfd is 50, key is test_row_0/A:col10/1731341661115/Put/seqid=0 2024-11-11T16:14:22,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741870_1046 (size=12151) 2024-11-11T16:14:23,003 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/6c1072d91e184161867a09c540646cfd 2024-11-11T16:14:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/7855259c3e9542b7be7e89ac0cbdf3f2 is 50, key is test_row_0/B:col10/1731341661115/Put/seqid=0 2024-11-11T16:14:23,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741871_1047 (size=12151) 2024-11-11T16:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:23,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:23,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341723312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341723314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341723314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341723314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341723320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341723426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,430 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/7855259c3e9542b7be7e89ac0cbdf3f2 2024-11-11T16:14:23,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341723426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341723426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341723426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341723436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/f6ba42aa934c4f25bceb89cc2a8a0785 is 50, key is test_row_0/C:col10/1731341661115/Put/seqid=0 2024-11-11T16:14:23,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741872_1048 (size=12151) 2024-11-11T16:14:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T16:14:23,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341723636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341723636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341723637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341723646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341723646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,866 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/f6ba42aa934c4f25bceb89cc2a8a0785 2024-11-11T16:14:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/6c1072d91e184161867a09c540646cfd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6c1072d91e184161867a09c540646cfd 2024-11-11T16:14:23,883 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6c1072d91e184161867a09c540646cfd, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T16:14:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/7855259c3e9542b7be7e89ac0cbdf3f2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7855259c3e9542b7be7e89ac0cbdf3f2 2024-11-11T16:14:23,892 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7855259c3e9542b7be7e89ac0cbdf3f2, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T16:14:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/f6ba42aa934c4f25bceb89cc2a8a0785 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/f6ba42aa934c4f25bceb89cc2a8a0785 2024-11-11T16:14:23,900 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/f6ba42aa934c4f25bceb89cc2a8a0785, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T16:14:23,901 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for af9dedd7afe35ed4a9eb53f0210465b1 in 1315ms, sequenceid=158, compaction requested=true 2024-11-11T16:14:23,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:23,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T16:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T16:14:23,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T16:14:23,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5830 sec 2024-11-11T16:14:23,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.5900 sec 2024-11-11T16:14:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:23,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T16:14:23,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:23,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:23,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:23,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:23,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:23,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:23,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/2561f149ecb9453f8234e07e9cb65eb9 is 50, key is test_row_0/A:col10/1731341663955/Put/seqid=0 2024-11-11T16:14:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741873_1049 (size=19321) 2024-11-11T16:14:23,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/2561f149ecb9453f8234e07e9cb65eb9 2024-11-11T16:14:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341723971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341723974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341723974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/53090d875acc4e73af992c7c23297d43 is 50, key is test_row_0/B:col10/1731341663955/Put/seqid=0 2024-11-11T16:14:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341723978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:23,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741874_1050 (size=12151) 2024-11-11T16:14:23,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/53090d875acc4e73af992c7c23297d43 2024-11-11T16:14:24,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341723981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/986025dcf23445e6bb46a59309a81a7c is 50, key is test_row_0/C:col10/1731341663955/Put/seqid=0 2024-11-11T16:14:24,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741875_1051 (size=12151) 2024-11-11T16:14:24,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/986025dcf23445e6bb46a59309a81a7c 2024-11-11T16:14:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/2561f149ecb9453f8234e07e9cb65eb9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/2561f149ecb9453f8234e07e9cb65eb9 2024-11-11T16:14:24,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/2561f149ecb9453f8234e07e9cb65eb9, entries=300, sequenceid=176, filesize=18.9 K 2024-11-11T16:14:24,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/53090d875acc4e73af992c7c23297d43 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/53090d875acc4e73af992c7c23297d43 2024-11-11T16:14:24,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/53090d875acc4e73af992c7c23297d43, entries=150, sequenceid=176, filesize=11.9 K 2024-11-11T16:14:24,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/986025dcf23445e6bb46a59309a81a7c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/986025dcf23445e6bb46a59309a81a7c 2024-11-11T16:14:24,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341724088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341724088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341724086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/986025dcf23445e6bb46a59309a81a7c, entries=150, sequenceid=176, filesize=11.9 K 2024-11-11T16:14:24,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for af9dedd7afe35ed4a9eb53f0210465b1 in 144ms, sequenceid=176, compaction requested=true 2024-11-11T16:14:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:24,100 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:14:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:24,100 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:14:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:24,105 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58322 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:14:24,105 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:24,105 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:24,105 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/4d7bed22c2b0435eb9388e681a80513a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cec164d9003142498f850f360bd68b34, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6c1072d91e184161867a09c540646cfd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/2561f149ecb9453f8234e07e9cb65eb9] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=57.0 K 2024-11-11T16:14:24,106 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:14:24,106 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:24,106 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:24,106 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f05507cdb085418f8cb2d7ef6c6d33fe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/3d85ca404c2b4580986d9581334450ca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7855259c3e9542b7be7e89ac0cbdf3f2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/53090d875acc4e73af992c7c23297d43] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=47.6 K 2024-11-11T16:14:24,107 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f05507cdb085418f8cb2d7ef6c6d33fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731341660406 2024-11-11T16:14:24,107 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7bed22c2b0435eb9388e681a80513a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731341660406 2024-11-11T16:14:24,108 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec164d9003142498f850f360bd68b34, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731341660428 2024-11-11T16:14:24,108 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d85ca404c2b4580986d9581334450ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731341660437 2024-11-11T16:14:24,109 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7855259c3e9542b7be7e89ac0cbdf3f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731341661106 2024-11-11T16:14:24,109 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c1072d91e184161867a09c540646cfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731341661106 2024-11-11T16:14:24,110 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 53090d875acc4e73af992c7c23297d43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731341663311 2024-11-11T16:14:24,110 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2561f149ecb9453f8234e07e9cb65eb9, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731341663300 2024-11-11T16:14:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:24,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:14:24,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:24,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:24,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:24,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:24,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:24,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:24,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/1d3c82780f0e4945ac52c0f55ad99de0 is 50, key is test_row_0/A:col10/1731341664110/Put/seqid=0 2024-11-11T16:14:24,147 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:24,148 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/6672c70f317545cd969af9d0db660365 is 50, key is test_row_0/A:col10/1731341663955/Put/seqid=0 2024-11-11T16:14:24,156 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#41 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:24,157 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/ef560d922c81467eb8ee7cd5b6b05a25 is 50, key is test_row_0/B:col10/1731341663955/Put/seqid=0 2024-11-11T16:14:24,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741877_1053 (size=12595) 2024-11-11T16:14:24,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741876_1052 (size=12151) 2024-11-11T16:14:24,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/1d3c82780f0e4945ac52c0f55ad99de0 2024-11-11T16:14:24,236 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/6672c70f317545cd969af9d0db660365 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6672c70f317545cd969af9d0db660365 2024-11-11T16:14:24,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741878_1054 (size=12595) 2024-11-11T16:14:24,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/143dc9c5df3c46cfa57d33719346ab94 is 50, key is test_row_0/B:col10/1731341664110/Put/seqid=0 2024-11-11T16:14:24,257 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/ef560d922c81467eb8ee7cd5b6b05a25 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/ef560d922c81467eb8ee7cd5b6b05a25 2024-11-11T16:14:24,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741879_1055 (size=12151) 2024-11-11T16:14:24,262 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 6672c70f317545cd969af9d0db660365(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:24,263 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:24,263 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=12, startTime=1731341664100; duration=0sec 2024-11-11T16:14:24,263 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:24,263 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:24,263 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:14:24,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/143dc9c5df3c46cfa57d33719346ab94 2024-11-11T16:14:24,269 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:14:24,269 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:24,270 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:24,270 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/9871d226eced4f4293a1d8a58564b368, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/dbc051e1978142bbb258b2d941f6226d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/f6ba42aa934c4f25bceb89cc2a8a0785, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/986025dcf23445e6bb46a59309a81a7c] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=47.6 K 2024-11-11T16:14:24,270 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9871d226eced4f4293a1d8a58564b368, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731341660406 2024-11-11T16:14:24,271 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbc051e1978142bbb258b2d941f6226d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731341660437 2024-11-11T16:14:24,272 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ba42aa934c4f25bceb89cc2a8a0785, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731341661106 2024-11-11T16:14:24,272 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 986025dcf23445e6bb46a59309a81a7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731341663311 2024-11-11T16:14:24,276 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into ef560d922c81467eb8ee7cd5b6b05a25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:24,276 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:24,276 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=12, startTime=1731341664100; duration=0sec 2024-11-11T16:14:24,277 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:24,277 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:24,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/8d58164573e94fd39cf6c473e67f93ae is 50, key is test_row_0/C:col10/1731341664110/Put/seqid=0 2024-11-11T16:14:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341724295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,303 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:24,304 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/40f6c5b95ce14f3480efaec56831c760 is 50, key is test_row_0/C:col10/1731341663955/Put/seqid=0 2024-11-11T16:14:24,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741880_1056 (size=12151) 2024-11-11T16:14:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341724299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341724308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341724308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341724308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741881_1057 (size=12595) 2024-11-11T16:14:24,347 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/40f6c5b95ce14f3480efaec56831c760 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/40f6c5b95ce14f3480efaec56831c760 2024-11-11T16:14:24,361 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into 40f6c5b95ce14f3480efaec56831c760(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:24,361 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:24,361 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=12, startTime=1731341664100; duration=0sec 2024-11-11T16:14:24,361 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:24,361 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:24,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341724405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341724416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341724616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341724626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341724626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341724627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341724627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:24,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/8d58164573e94fd39cf6c473e67f93ae 2024-11-11T16:14:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/1d3c82780f0e4945ac52c0f55ad99de0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1d3c82780f0e4945ac52c0f55ad99de0 2024-11-11T16:14:24,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1d3c82780f0e4945ac52c0f55ad99de0, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T16:14:24,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/143dc9c5df3c46cfa57d33719346ab94 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/143dc9c5df3c46cfa57d33719346ab94 2024-11-11T16:14:24,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/143dc9c5df3c46cfa57d33719346ab94, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T16:14:24,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/8d58164573e94fd39cf6c473e67f93ae as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/8d58164573e94fd39cf6c473e67f93ae 2024-11-11T16:14:24,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/8d58164573e94fd39cf6c473e67f93ae, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T16:14:24,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for af9dedd7afe35ed4a9eb53f0210465b1 in 677ms, sequenceid=191, compaction requested=false 2024-11-11T16:14:24,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:24,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T16:14:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:24,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/23027db177a14d2eaae579bc5eb83b62 is 50, key is test_row_0/A:col10/1731341664927/Put/seqid=0 2024-11-11T16:14:24,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741882_1058 (size=14541) 2024-11-11T16:14:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341725010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341725011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341725126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341725126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341725136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341725146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341725146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341725335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341725336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/23027db177a14d2eaae579bc5eb83b62 2024-11-11T16:14:25,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/cc703283a4d247f2a812f79a305321ee is 50, key is test_row_0/B:col10/1731341664927/Put/seqid=0 2024-11-11T16:14:25,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741883_1059 (size=12151) 2024-11-11T16:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T16:14:25,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:25,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T16:14:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T16:14:25,487 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:25,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:25,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T16:14:25,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T16:14:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:25,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341725646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341725652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:25,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:25,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/cc703283a4d247f2a812f79a305321ee 2024-11-11T16:14:25,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T16:14:25,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:25,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:25,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:25,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/2bb1129a10be45bc82bb3eb435e7dce0 is 50, key is test_row_0/C:col10/1731341664927/Put/seqid=0 2024-11-11T16:14:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T16:14:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741884_1060 (size=12151) 2024-11-11T16:14:25,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T16:14:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:25,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:26,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T16:14:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T16:14:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:26,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341726148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:26,154 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:26,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341726156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:26,165 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341726157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:26,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341726157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:26,170 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:26,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341726166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:26,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/2bb1129a10be45bc82bb3eb435e7dce0 2024-11-11T16:14:26,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/23027db177a14d2eaae579bc5eb83b62 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/23027db177a14d2eaae579bc5eb83b62 2024-11-11T16:14:26,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T16:14:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:26,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/23027db177a14d2eaae579bc5eb83b62, entries=200, sequenceid=212, filesize=14.2 K 2024-11-11T16:14:26,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/cc703283a4d247f2a812f79a305321ee as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/cc703283a4d247f2a812f79a305321ee 2024-11-11T16:14:26,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/cc703283a4d247f2a812f79a305321ee, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T16:14:26,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/2bb1129a10be45bc82bb3eb435e7dce0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/2bb1129a10be45bc82bb3eb435e7dce0 2024-11-11T16:14:26,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/2bb1129a10be45bc82bb3eb435e7dce0, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T16:14:26,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for af9dedd7afe35ed4a9eb53f0210465b1 in 1415ms, sequenceid=212, compaction requested=true 2024-11-11T16:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:26,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:26,345 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:26,346 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:26,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:26,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:26,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:26,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:26,350 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:26,350 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:26,350 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,351 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6672c70f317545cd969af9d0db660365, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1d3c82780f0e4945ac52c0f55ad99de0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/23027db177a14d2eaae579bc5eb83b62] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=38.4 K 2024-11-11T16:14:26,351 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:26,351 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:26,352 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,352 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/ef560d922c81467eb8ee7cd5b6b05a25, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/143dc9c5df3c46cfa57d33719346ab94, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/cc703283a4d247f2a812f79a305321ee] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.0 K 2024-11-11T16:14:26,352 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6672c70f317545cd969af9d0db660365, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731341663311 2024-11-11T16:14:26,353 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ef560d922c81467eb8ee7cd5b6b05a25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731341663311 2024-11-11T16:14:26,353 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3c82780f0e4945ac52c0f55ad99de0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731341663970 2024-11-11T16:14:26,354 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23027db177a14d2eaae579bc5eb83b62, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731341664169 2024-11-11T16:14:26,354 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 143dc9c5df3c46cfa57d33719346ab94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731341663970 2024-11-11T16:14:26,357 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting cc703283a4d247f2a812f79a305321ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731341664169 2024-11-11T16:14:26,387 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:26,388 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/807d6d9db12344bcbbd350764c348f4b is 50, key is test_row_0/B:col10/1731341664927/Put/seqid=0 2024-11-11T16:14:26,397 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:26,398 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f645e8d7951d4236b7641543f5d14066 is 50, key is test_row_0/A:col10/1731341664927/Put/seqid=0 2024-11-11T16:14:26,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741885_1061 (size=12697) 2024-11-11T16:14:26,419 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/807d6d9db12344bcbbd350764c348f4b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/807d6d9db12344bcbbd350764c348f4b 2024-11-11T16:14:26,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T16:14:26,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,431 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:14:26,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:26,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:26,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:26,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:26,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:26,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:26,435 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into 807d6d9db12344bcbbd350764c348f4b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:26,435 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:26,435 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341666345; duration=0sec 2024-11-11T16:14:26,435 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:26,436 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:26,436 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/52e8e706a45e48b6b120fc277a5e4d9c is 50, key is test_row_0/A:col10/1731341664993/Put/seqid=0 2024-11-11T16:14:26,442 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:26,442 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:26,442 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,443 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/40f6c5b95ce14f3480efaec56831c760, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/8d58164573e94fd39cf6c473e67f93ae, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/2bb1129a10be45bc82bb3eb435e7dce0] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.0 K 2024-11-11T16:14:26,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741886_1062 (size=12697) 2024-11-11T16:14:26,449 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f6c5b95ce14f3480efaec56831c760, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731341663311 2024-11-11T16:14:26,450 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d58164573e94fd39cf6c473e67f93ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731341663970 2024-11-11T16:14:26,451 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bb1129a10be45bc82bb3eb435e7dce0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731341664169 2024-11-11T16:14:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741887_1063 (size=12151) 2024-11-11T16:14:26,468 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:26,468 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/cd9ed7b454ff48aa8e967f0c64554237 is 50, key is test_row_0/C:col10/1731341664927/Put/seqid=0 2024-11-11T16:14:26,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741888_1064 (size=12697) 2024-11-11T16:14:26,510 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/cd9ed7b454ff48aa8e967f0c64554237 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cd9ed7b454ff48aa8e967f0c64554237 2024-11-11T16:14:26,529 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into cd9ed7b454ff48aa8e967f0c64554237(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:26,529 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:26,529 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341666346; duration=0sec 2024-11-11T16:14:26,529 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:26,529 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T16:14:26,869 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/52e8e706a45e48b6b120fc277a5e4d9c 2024-11-11T16:14:26,873 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f645e8d7951d4236b7641543f5d14066 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f645e8d7951d4236b7641543f5d14066 2024-11-11T16:14:26,885 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into f645e8d7951d4236b7641543f5d14066(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:26,885 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:26,885 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341666345; duration=0sec 2024-11-11T16:14:26,885 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:26,885 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/c198802167674de9a11ce2f30fa42c1c is 50, key is test_row_0/B:col10/1731341664993/Put/seqid=0 2024-11-11T16:14:26,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741889_1065 (size=12151) 2024-11-11T16:14:26,900 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/c198802167674de9a11ce2f30fa42c1c 2024-11-11T16:14:26,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/39c3058cf32c41a59521eef9a8742062 is 50, key is test_row_0/C:col10/1731341664993/Put/seqid=0 2024-11-11T16:14:26,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741890_1066 (size=12151) 2024-11-11T16:14:26,963 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/39c3058cf32c41a59521eef9a8742062 2024-11-11T16:14:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/52e8e706a45e48b6b120fc277a5e4d9c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/52e8e706a45e48b6b120fc277a5e4d9c 2024-11-11T16:14:26,979 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/52e8e706a45e48b6b120fc277a5e4d9c, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T16:14:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/c198802167674de9a11ce2f30fa42c1c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/c198802167674de9a11ce2f30fa42c1c 2024-11-11T16:14:26,988 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/c198802167674de9a11ce2f30fa42c1c, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T16:14:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/39c3058cf32c41a59521eef9a8742062 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/39c3058cf32c41a59521eef9a8742062 2024-11-11T16:14:26,997 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/39c3058cf32c41a59521eef9a8742062, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T16:14:26,998 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for af9dedd7afe35ed4a9eb53f0210465b1 in 568ms, sequenceid=228, compaction requested=false 2024-11-11T16:14:26,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:26,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:26,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T16:14:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T16:14:27,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T16:14:27,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5130 sec 2024-11-11T16:14:27,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5280 sec 2024-11-11T16:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:27,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:14:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:27,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/57a7b20a064b415bbbd84c6f9a7e686f is 50, key is test_row_0/A:col10/1731341667203/Put/seqid=0 2024-11-11T16:14:27,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741891_1067 (size=12147) 2024-11-11T16:14:27,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341667207, max timestamp=1731341667276Num uniques 859; 2024-11-11T16:14:27,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341667207, max timestamp=1731341667276Num uniques 859; 2024-11-11T16:14:27,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:14:27,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:14:27,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341667267, max timestamp=1731341667276Num uniques 859; 2024-11-11T16:14:27,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:14:27,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341727371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:27,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341727375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:27,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:27,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341727486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:27,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341727496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T16:14:27,636 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:27,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/57a7b20a064b415bbbd84c6f9a7e686f 2024-11-11T16:14:27,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T16:14:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T16:14:27,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:27,640 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:27,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:27,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/09ca7d0968354d9b90a0e8f1c143f75e is 50, key is test_row_0/B:col10/1731341667203/Put/seqid=0 2024-11-11T16:14:27,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741892_1068 (size=9757) 2024-11-11T16:14:27,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/09ca7d0968354d9b90a0e8f1c143f75e 2024-11-11T16:14:27,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a0e9471e7ecb4f198a002c05cdd8b0ba is 50, key is test_row_0/C:col10/1731341667203/Put/seqid=0 2024-11-11T16:14:27,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741893_1069 (size=9757) 2024-11-11T16:14:27,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341727706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:27,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a0e9471e7ecb4f198a002c05cdd8b0ba 2024-11-11T16:14:27,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341727716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:27,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:27,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/57a7b20a064b415bbbd84c6f9a7e686f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/57a7b20a064b415bbbd84c6f9a7e686f 2024-11-11T16:14:27,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/57a7b20a064b415bbbd84c6f9a7e686f, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T16:14:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/09ca7d0968354d9b90a0e8f1c143f75e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/09ca7d0968354d9b90a0e8f1c143f75e 2024-11-11T16:14:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T16:14:27,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/09ca7d0968354d9b90a0e8f1c143f75e, entries=100, sequenceid=240, filesize=9.5 K 2024-11-11T16:14:27,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a0e9471e7ecb4f198a002c05cdd8b0ba as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a0e9471e7ecb4f198a002c05cdd8b0ba 2024-11-11T16:14:27,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a0e9471e7ecb4f198a002c05cdd8b0ba, entries=100, sequenceid=240, filesize=9.5 K 2024-11-11T16:14:27,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for af9dedd7afe35ed4a9eb53f0210465b1 in 575ms, sequenceid=240, compaction requested=true 2024-11-11T16:14:27,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:27,782 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:27,782 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:27,784 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36995 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:27,784 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:27,784 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:27,784 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:27,784 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:27,784 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:27,784 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f645e8d7951d4236b7641543f5d14066, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/52e8e706a45e48b6b120fc277a5e4d9c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/57a7b20a064b415bbbd84c6f9a7e686f] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.1 K 2024-11-11T16:14:27,784 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/807d6d9db12344bcbbd350764c348f4b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/c198802167674de9a11ce2f30fa42c1c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/09ca7d0968354d9b90a0e8f1c143f75e] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=33.8 K 2024-11-11T16:14:27,785 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f645e8d7951d4236b7641543f5d14066, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731341664169 2024-11-11T16:14:27,785 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 807d6d9db12344bcbbd350764c348f4b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731341664169 2024-11-11T16:14:27,786 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e8e706a45e48b6b120fc277a5e4d9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731341664993 2024-11-11T16:14:27,787 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c198802167674de9a11ce2f30fa42c1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731341664993 2024-11-11T16:14:27,787 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a7b20a064b415bbbd84c6f9a7e686f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731341667193 2024-11-11T16:14:27,789 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 09ca7d0968354d9b90a0e8f1c143f75e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731341667193 2024-11-11T16:14:27,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T16:14:27,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:27,795 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-11T16:14:27,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:27,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:27,810 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:27,811 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/79b6edbc61354308a2023d57df5aad02 is 50, key is test_row_0/A:col10/1731341667203/Put/seqid=0 2024-11-11T16:14:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/66eb546ea3f7421f9753cacc3d90886c is 50, key is test_row_0/A:col10/1731341667366/Put/seqid=0 2024-11-11T16:14:27,825 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:27,826 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/e57f5703b38948819b439d41a5092ae6 is 50, key is test_row_0/B:col10/1731341667203/Put/seqid=0 2024-11-11T16:14:27,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741895_1071 (size=12301) 2024-11-11T16:14:27,853 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/66eb546ea3f7421f9753cacc3d90886c 2024-11-11T16:14:27,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741896_1072 (size=12799) 2024-11-11T16:14:27,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741894_1070 (size=12799) 2024-11-11T16:14:27,868 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/e57f5703b38948819b439d41a5092ae6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/e57f5703b38948819b439d41a5092ae6 2024-11-11T16:14:27,877 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into e57f5703b38948819b439d41a5092ae6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:27,877 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:27,877 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341667782; duration=0sec 2024-11-11T16:14:27,878 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:27,878 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:27,878 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:27,879 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:27,879 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:27,879 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:27,881 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cd9ed7b454ff48aa8e967f0c64554237, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/39c3058cf32c41a59521eef9a8742062, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a0e9471e7ecb4f198a002c05cdd8b0ba] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=33.8 K 2024-11-11T16:14:27,881 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9ed7b454ff48aa8e967f0c64554237, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731341664169 2024-11-11T16:14:27,882 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c3058cf32c41a59521eef9a8742062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731341664993 2024-11-11T16:14:27,882 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a0e9471e7ecb4f198a002c05cdd8b0ba, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731341667193 2024-11-11T16:14:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f6ddcf6dd2d6484cbf517041a49d69b7 is 50, key is test_row_0/B:col10/1731341667366/Put/seqid=0 2024-11-11T16:14:27,907 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:27,908 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/ff744e3cee6544ae96381df17ccb05c1 is 50, key is test_row_0/C:col10/1731341667203/Put/seqid=0 2024-11-11T16:14:27,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741897_1073 (size=12301) 2024-11-11T16:14:27,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741898_1074 (size=12799) 2024-11-11T16:14:27,946 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/ff744e3cee6544ae96381df17ccb05c1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ff744e3cee6544ae96381df17ccb05c1 2024-11-11T16:14:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T16:14:27,959 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into ff744e3cee6544ae96381df17ccb05c1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:27,959 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:27,959 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341667782; duration=0sec 2024-11-11T16:14:27,959 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:27,959 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:28,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:28,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341728040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341728042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341728155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341728155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341728176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,180 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341728185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,191 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341728196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,201 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T16:14:28,267 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/79b6edbc61354308a2023d57df5aad02 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/79b6edbc61354308a2023d57df5aad02 2024-11-11T16:14:28,274 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 79b6edbc61354308a2023d57df5aad02(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:28,274 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:28,274 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341667782; duration=0sec 2024-11-11T16:14:28,275 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:28,275 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:28,320 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f6ddcf6dd2d6484cbf517041a49d69b7 2024-11-11T16:14:28,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/323743b7f8a0496ca35e5fba0a5982ce is 50, key is test_row_0/C:col10/1731341667366/Put/seqid=0 2024-11-11T16:14:28,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741899_1075 (size=12301) 2024-11-11T16:14:28,343 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/323743b7f8a0496ca35e5fba0a5982ce 2024-11-11T16:14:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/66eb546ea3f7421f9753cacc3d90886c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/66eb546ea3f7421f9753cacc3d90886c 2024-11-11T16:14:28,359 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/66eb546ea3f7421f9753cacc3d90886c, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T16:14:28,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f6ddcf6dd2d6484cbf517041a49d69b7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f6ddcf6dd2d6484cbf517041a49d69b7 2024-11-11T16:14:28,370 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f6ddcf6dd2d6484cbf517041a49d69b7, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T16:14:28,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341728369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341728369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/323743b7f8a0496ca35e5fba0a5982ce as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/323743b7f8a0496ca35e5fba0a5982ce 2024-11-11T16:14:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,388 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/323743b7f8a0496ca35e5fba0a5982ce, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T16:14:28,390 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for af9dedd7afe35ed4a9eb53f0210465b1 in 595ms, sequenceid=276, compaction requested=false 2024-11-11T16:14:28,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:28,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:28,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T16:14:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T16:14:28,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T16:14:28,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-11T16:14:28,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 760 msec 2024-11-11T16:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:28,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T16:14:28,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:28,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:28,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:28,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:28,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:28,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:28,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/152a9a1364154ea582591bfe48f20808 is 50, key is test_row_0/A:col10/1731341668035/Put/seqid=0 2024-11-11T16:14:28,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741900_1076 (size=14741) 2024-11-11T16:14:28,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/152a9a1364154ea582591bfe48f20808 2024-11-11T16:14:28,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/7668c133b53f4147b5ddac731d949c7d is 50, key is test_row_0/B:col10/1731341668035/Put/seqid=0 2024-11-11T16:14:28,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341668745, max timestamp=1731341668762Num uniques 859; 2024-11-11T16:14:28,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:14:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T16:14:28,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341668745, max timestamp=1731341668766Num uniques 859; 2024-11-11T16:14:28,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:14:28,777 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:28,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:28,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741901_1077 (size=12301) 2024-11-11T16:14:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T16:14:28,780 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T16:14:28,782 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:28,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:28,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341668745, max timestamp=1731341668762Num uniques 859; 2024-11-11T16:14:28,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T16:14:28,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341728841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341728848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T16:14:28,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T16:14:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:28,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:28,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341728956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:28,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341728966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T16:14:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T16:14:29,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/7668c133b53f4147b5ddac731d949c7d 2024-11-11T16:14:29,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341729176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341729177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/80666943af17471c82521430d21b8a85 is 50, key is test_row_0/C:col10/1731341668035/Put/seqid=0 2024-11-11T16:14:29,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T16:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:29,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741902_1078 (size=12301) 2024-11-11T16:14:29,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/80666943af17471c82521430d21b8a85 2024-11-11T16:14:29,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/152a9a1364154ea582591bfe48f20808 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/152a9a1364154ea582591bfe48f20808 2024-11-11T16:14:29,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/152a9a1364154ea582591bfe48f20808, entries=200, sequenceid=290, filesize=14.4 K 2024-11-11T16:14:29,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/7668c133b53f4147b5ddac731d949c7d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7668c133b53f4147b5ddac731d949c7d 2024-11-11T16:14:29,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7668c133b53f4147b5ddac731d949c7d, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T16:14:29,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/80666943af17471c82521430d21b8a85 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/80666943af17471c82521430d21b8a85 2024-11-11T16:14:29,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/80666943af17471c82521430d21b8a85, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T16:14:29,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for af9dedd7afe35ed4a9eb53f0210465b1 in 647ms, sequenceid=290, compaction requested=true 2024-11-11T16:14:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:29,327 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:29,327 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:29,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:29,329 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:29,329 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:29,329 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:29,329 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:29,329 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,329 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,329 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/79b6edbc61354308a2023d57df5aad02, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/66eb546ea3f7421f9753cacc3d90886c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/152a9a1364154ea582591bfe48f20808] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=38.9 K 2024-11-11T16:14:29,330 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/e57f5703b38948819b439d41a5092ae6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f6ddcf6dd2d6484cbf517041a49d69b7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7668c133b53f4147b5ddac731d949c7d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.5 K 2024-11-11T16:14:29,330 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b6edbc61354308a2023d57df5aad02, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731341665006 2024-11-11T16:14:29,330 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e57f5703b38948819b439d41a5092ae6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731341665006 2024-11-11T16:14:29,331 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ddcf6dd2d6484cbf517041a49d69b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731341667353 2024-11-11T16:14:29,331 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66eb546ea3f7421f9753cacc3d90886c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731341667353 2024-11-11T16:14:29,331 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7668c133b53f4147b5ddac731d949c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731341668019 2024-11-11T16:14:29,332 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152a9a1364154ea582591bfe48f20808, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731341668019 2024-11-11T16:14:29,346 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:29,347 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/6e5d8a1e095f47e5b6a517d9a67bf95f is 50, key is test_row_0/B:col10/1731341668035/Put/seqid=0 2024-11-11T16:14:29,358 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:29,359 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/be8e6fe71aeb46edb5aedd8bd7737243 is 50, key is test_row_0/A:col10/1731341668035/Put/seqid=0 2024-11-11T16:14:29,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741903_1079 (size=13051) 2024-11-11T16:14:29,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741904_1080 (size=13051) 2024-11-11T16:14:29,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T16:14:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,400 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-11T16:14:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:29,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:29,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:29,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:29,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:29,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T16:14:29,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/cfd7cc8e15154e89b0c0cb9a66c15451 is 50, key is test_row_0/A:col10/1731341668835/Put/seqid=0 2024-11-11T16:14:29,420 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/6e5d8a1e095f47e5b6a517d9a67bf95f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6e5d8a1e095f47e5b6a517d9a67bf95f 2024-11-11T16:14:29,429 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into 6e5d8a1e095f47e5b6a517d9a67bf95f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:29,429 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341669327; duration=0sec 2024-11-11T16:14:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:29,431 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:29,431 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:29,431 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,431 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ff744e3cee6544ae96381df17ccb05c1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/323743b7f8a0496ca35e5fba0a5982ce, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/80666943af17471c82521430d21b8a85] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.5 K 2024-11-11T16:14:29,432 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ff744e3cee6544ae96381df17ccb05c1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731341665006 2024-11-11T16:14:29,432 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 323743b7f8a0496ca35e5fba0a5982ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731341667353 2024-11-11T16:14:29,433 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 80666943af17471c82521430d21b8a85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731341668019 2024-11-11T16:14:29,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741905_1081 (size=12301) 2024-11-11T16:14:29,469 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/cfd7cc8e15154e89b0c0cb9a66c15451 2024-11-11T16:14:29,484 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:29,485 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/375ab23f9b264075b39298afcb65358b is 50, key is test_row_0/C:col10/1731341668035/Put/seqid=0 2024-11-11T16:14:29,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/07febddf1c544495a4b57b9b59676cf7 is 50, key is test_row_0/B:col10/1731341668835/Put/seqid=0 2024-11-11T16:14:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:29,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741906_1082 (size=13051) 2024-11-11T16:14:29,556 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/375ab23f9b264075b39298afcb65358b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/375ab23f9b264075b39298afcb65358b 2024-11-11T16:14:29,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741907_1083 (size=12301) 2024-11-11T16:14:29,586 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/07febddf1c544495a4b57b9b59676cf7 2024-11-11T16:14:29,592 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into 375ab23f9b264075b39298afcb65358b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:29,592 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:29,592 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341669327; duration=0sec 2024-11-11T16:14:29,592 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:29,592 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:29,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a8e72f84ead74286931d28b8e62a6747 is 50, key is test_row_0/C:col10/1731341668835/Put/seqid=0 2024-11-11T16:14:29,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341729594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341729594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741908_1084 (size=12301) 2024-11-11T16:14:29,620 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a8e72f84ead74286931d28b8e62a6747 2024-11-11T16:14:29,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/cfd7cc8e15154e89b0c0cb9a66c15451 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cfd7cc8e15154e89b0c0cb9a66c15451 2024-11-11T16:14:29,639 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cfd7cc8e15154e89b0c0cb9a66c15451, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T16:14:29,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/07febddf1c544495a4b57b9b59676cf7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/07febddf1c544495a4b57b9b59676cf7 2024-11-11T16:14:29,654 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/07febddf1c544495a4b57b9b59676cf7, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T16:14:29,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T16:14:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/a8e72f84ead74286931d28b8e62a6747 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a8e72f84ead74286931d28b8e62a6747 2024-11-11T16:14:29,673 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a8e72f84ead74286931d28b8e62a6747, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T16:14:29,681 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=60.38 KB/61830 for af9dedd7afe35ed4a9eb53f0210465b1 in 281ms, sequenceid=326, compaction requested=false 2024-11-11T16:14:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T16:14:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T16:14:29,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 921 msec 2024-11-11T16:14:29,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T16:14:29,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 907 msec 2024-11-11T16:14:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:29,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:14:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:29,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/30d775fe287f4565b1ec41b7540032e0 is 50, key is test_row_0/A:col10/1731341669582/Put/seqid=0 2024-11-11T16:14:29,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741909_1085 (size=12301) 2024-11-11T16:14:29,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/30d775fe287f4565b1ec41b7540032e0 2024-11-11T16:14:29,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341729783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341729792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,807 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/be8e6fe71aeb46edb5aedd8bd7737243 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/be8e6fe71aeb46edb5aedd8bd7737243 2024-11-11T16:14:29,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/0a65cb6c732b415dab6735ea2f59f41a is 50, key is test_row_0/B:col10/1731341669582/Put/seqid=0 2024-11-11T16:14:29,841 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into be8e6fe71aeb46edb5aedd8bd7737243(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:29,841 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:29,841 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341669327; duration=0sec 2024-11-11T16:14:29,841 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:29,841 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:29,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741910_1086 (size=12301) 2024-11-11T16:14:29,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/0a65cb6c732b415dab6735ea2f59f41a 2024-11-11T16:14:29,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341729896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T16:14:29,916 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:29,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T16:14:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T16:14:29,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:29,924 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:29,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:29,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341729911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:29,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/df49c5819c79463eb750b63ca18e631d is 50, key is test_row_0/C:col10/1731341669582/Put/seqid=0 2024-11-11T16:14:29,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741911_1087 (size=12301) 2024-11-11T16:14:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T16:14:30,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341730106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:30,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341730136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T16:14:30,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:30,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:30,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/df49c5819c79463eb750b63ca18e631d 2024-11-11T16:14:30,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:30,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:30,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/30d775fe287f4565b1ec41b7540032e0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/30d775fe287f4565b1ec41b7540032e0 2024-11-11T16:14:30,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341730426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:30,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/30d775fe287f4565b1ec41b7540032e0, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T16:14:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/0a65cb6c732b415dab6735ea2f59f41a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/0a65cb6c732b415dab6735ea2f59f41a 2024-11-11T16:14:30,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/0a65cb6c732b415dab6735ea2f59f41a, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T16:14:30,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/df49c5819c79463eb750b63ca18e631d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/df49c5819c79463eb750b63ca18e631d 2024-11-11T16:14:30,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/df49c5819c79463eb750b63ca18e631d, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T16:14:30,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for af9dedd7afe35ed4a9eb53f0210465b1 in 757ms, sequenceid=341, compaction requested=true 2024-11-11T16:14:30,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:30,477 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:30,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:30,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:30,477 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:30,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:30,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:30,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:30,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:30,479 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:30,479 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:30,479 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,479 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/be8e6fe71aeb46edb5aedd8bd7737243, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cfd7cc8e15154e89b0c0cb9a66c15451, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/30d775fe287f4565b1ec41b7540032e0] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.8 K 2024-11-11T16:14:30,480 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:30,480 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:30,480 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,480 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6e5d8a1e095f47e5b6a517d9a67bf95f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/07febddf1c544495a4b57b9b59676cf7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/0a65cb6c732b415dab6735ea2f59f41a] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.8 K 2024-11-11T16:14:30,480 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting be8e6fe71aeb46edb5aedd8bd7737243, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731341668019 2024-11-11T16:14:30,481 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e5d8a1e095f47e5b6a517d9a67bf95f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731341668019 2024-11-11T16:14:30,481 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfd7cc8e15154e89b0c0cb9a66c15451, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731341668826 2024-11-11T16:14:30,482 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 07febddf1c544495a4b57b9b59676cf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731341668826 2024-11-11T16:14:30,482 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d775fe287f4565b1ec41b7540032e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341669548 2024-11-11T16:14:30,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:14:30,483 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a65cb6c732b415dab6735ea2f59f41a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341669548 2024-11-11T16:14:30,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:30,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:30,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:30,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:30,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:30,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:30,503 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:30,504 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/14d2dc098def4c9fa040a7b84540d33e is 50, key is test_row_0/A:col10/1731341669582/Put/seqid=0 2024-11-11T16:14:30,516 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:30,517 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/37425e2f3331446185008d7ab6e324aa is 50, key is test_row_0/B:col10/1731341669582/Put/seqid=0 2024-11-11T16:14:30,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f6ff48e2af424ab1a670095f5763e8f1 is 50, key is test_row_0/A:col10/1731341669779/Put/seqid=0 2024-11-11T16:14:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T16:14:30,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341730573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741912_1088 (size=13153) 2024-11-11T16:14:30,616 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/14d2dc098def4c9fa040a7b84540d33e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/14d2dc098def4c9fa040a7b84540d33e 2024-11-11T16:14:30,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741913_1089 (size=13153) 2024-11-11T16:14:30,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741914_1090 (size=14741) 2024-11-11T16:14:30,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f6ff48e2af424ab1a670095f5763e8f1 2024-11-11T16:14:30,649 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/37425e2f3331446185008d7ab6e324aa as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/37425e2f3331446185008d7ab6e324aa 2024-11-11T16:14:30,653 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 14d2dc098def4c9fa040a7b84540d33e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:30,653 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:30,653 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341670477; duration=0sec 2024-11-11T16:14:30,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/2349e9e1d72740f194f61572ce15d8ae is 50, key is test_row_0/B:col10/1731341669779/Put/seqid=0 2024-11-11T16:14:30,660 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:30,661 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:30,661 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:30,662 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:30,662 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:30,662 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,662 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/375ab23f9b264075b39298afcb65358b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a8e72f84ead74286931d28b8e62a6747, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/df49c5819c79463eb750b63ca18e631d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.8 K 2024-11-11T16:14:30,663 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 375ab23f9b264075b39298afcb65358b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731341668019 2024-11-11T16:14:30,663 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e72f84ead74286931d28b8e62a6747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731341668826 2024-11-11T16:14:30,664 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting df49c5819c79463eb750b63ca18e631d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341669548 2024-11-11T16:14:30,665 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into 37425e2f3331446185008d7ab6e324aa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:30,665 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:30,665 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341670477; duration=0sec 2024-11-11T16:14:30,666 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:30,666 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:30,691 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341730686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:30,693 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/437ed8c719f7475d915e338456d73f54 is 50, key is test_row_0/C:col10/1731341669582/Put/seqid=0 2024-11-11T16:14:30,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:30,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:30,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741915_1091 (size=12301) 2024-11-11T16:14:30,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/2349e9e1d72740f194f61572ce15d8ae 2024-11-11T16:14:30,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741916_1092 (size=13153) 2024-11-11T16:14:30,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d55eb47e689e4d23a8023a875260c272 is 50, key is test_row_0/C:col10/1731341669779/Put/seqid=0 2024-11-11T16:14:30,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:30,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:30,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741917_1093 (size=12301) 2024-11-11T16:14:30,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d55eb47e689e4d23a8023a875260c272 2024-11-11T16:14:30,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341730905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:30,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f6ff48e2af424ab1a670095f5763e8f1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f6ff48e2af424ab1a670095f5763e8f1 2024-11-11T16:14:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:30,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f6ff48e2af424ab1a670095f5763e8f1, entries=200, sequenceid=362, filesize=14.4 K 2024-11-11T16:14:30,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/2349e9e1d72740f194f61572ce15d8ae as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/2349e9e1d72740f194f61572ce15d8ae 2024-11-11T16:14:30,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/2349e9e1d72740f194f61572ce15d8ae, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T16:14:30,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d55eb47e689e4d23a8023a875260c272 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d55eb47e689e4d23a8023a875260c272 2024-11-11T16:14:30,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d55eb47e689e4d23a8023a875260c272, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T16:14:30,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for af9dedd7afe35ed4a9eb53f0210465b1 in 469ms, sequenceid=362, compaction requested=false 2024-11-11T16:14:30,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:30,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:14:30,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:30,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:30,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:30,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:30,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:30,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:30,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/eeaab9b4ef77415985ad739a1ae4af10 is 50, key is test_row_0/A:col10/1731341670953/Put/seqid=0 2024-11-11T16:14:31,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741918_1094 (size=14741) 2024-11-11T16:14:31,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/eeaab9b4ef77415985ad739a1ae4af10 2024-11-11T16:14:31,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:31,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:31,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/024c0c2a703e4b379b03ce1bb1df568e is 50, key is test_row_0/B:col10/1731341670953/Put/seqid=0 2024-11-11T16:14:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T16:14:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741919_1095 (size=12301) 2024-11-11T16:14:31,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/024c0c2a703e4b379b03ce1bb1df568e 2024-11-11T16:14:31,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/cba89fc9eb05420d95c882197aba5bc8 is 50, key is test_row_0/C:col10/1731341670953/Put/seqid=0 2024-11-11T16:14:31,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341731145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:31,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741920_1096 (size=12301) 2024-11-11T16:14:31,192 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/437ed8c719f7475d915e338456d73f54 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/437ed8c719f7475d915e338456d73f54 2024-11-11T16:14:31,206 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into 437ed8c719f7475d915e338456d73f54(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:31,207 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:31,207 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341670478; duration=0sec 2024-11-11T16:14:31,207 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:31,207 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:31,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341731226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341731256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:31,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341731473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:31,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:31,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:31,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/cba89fc9eb05420d95c882197aba5bc8 2024-11-11T16:14:31,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/eeaab9b4ef77415985ad739a1ae4af10 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/eeaab9b4ef77415985ad739a1ae4af10 2024-11-11T16:14:31,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/eeaab9b4ef77415985ad739a1ae4af10, entries=200, sequenceid=377, filesize=14.4 K 2024-11-11T16:14:31,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/024c0c2a703e4b379b03ce1bb1df568e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/024c0c2a703e4b379b03ce1bb1df568e 2024-11-11T16:14:31,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/024c0c2a703e4b379b03ce1bb1df568e, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T16:14:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/cba89fc9eb05420d95c882197aba5bc8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cba89fc9eb05420d95c882197aba5bc8 2024-11-11T16:14:31,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cba89fc9eb05420d95c882197aba5bc8, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T16:14:31,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for af9dedd7afe35ed4a9eb53f0210465b1 in 691ms, sequenceid=377, compaction requested=true 2024-11-11T16:14:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:31,646 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:14:31,646 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:31,651 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:31,652 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:31,652 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:31,652 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:31,652 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,652 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,652 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/14d2dc098def4c9fa040a7b84540d33e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f6ff48e2af424ab1a670095f5763e8f1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/eeaab9b4ef77415985ad739a1ae4af10] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=41.6 K 2024-11-11T16:14:31,652 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/37425e2f3331446185008d7ab6e324aa, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/2349e9e1d72740f194f61572ce15d8ae, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/024c0c2a703e4b379b03ce1bb1df568e] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.9 K 2024-11-11T16:14:31,652 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d2dc098def4c9fa040a7b84540d33e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341669548 2024-11-11T16:14:31,653 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37425e2f3331446185008d7ab6e324aa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341669548 2024-11-11T16:14:31,653 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ff48e2af424ab1a670095f5763e8f1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731341669765 2024-11-11T16:14:31,654 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2349e9e1d72740f194f61572ce15d8ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731341669779 2024-11-11T16:14:31,654 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting eeaab9b4ef77415985ad739a1ae4af10, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731341670489 2024-11-11T16:14:31,655 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024c0c2a703e4b379b03ce1bb1df568e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731341670566 2024-11-11T16:14:31,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T16:14:31,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,661 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:14:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:31,673 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:31,674 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/954d5abeabaf4d77ac65286e5ba9eaab is 50, key is test_row_0/A:col10/1731341670953/Put/seqid=0 2024-11-11T16:14:31,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b24fc372b6704f27949308b858aae1c8 is 50, key is test_row_0/A:col10/1731341671127/Put/seqid=0 2024-11-11T16:14:31,692 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#85 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:31,698 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/829b21f5332a497bad2c0759f6d1f367 is 50, key is test_row_0/B:col10/1731341670953/Put/seqid=0 2024-11-11T16:14:31,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741921_1097 (size=13255) 2024-11-11T16:14:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:31,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741923_1099 (size=13255) 2024-11-11T16:14:31,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:31,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741922_1098 (size=12301) 2024-11-11T16:14:31,752 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b24fc372b6704f27949308b858aae1c8 2024-11-11T16:14:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f1e1099f6cbb40158f9a53e48c49b3c9 is 50, key is test_row_0/B:col10/1731341671127/Put/seqid=0 2024-11-11T16:14:31,773 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/829b21f5332a497bad2c0759f6d1f367 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/829b21f5332a497bad2c0759f6d1f367 2024-11-11T16:14:31,781 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into 829b21f5332a497bad2c0759f6d1f367(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:31,781 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:31,781 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341671646; duration=0sec 2024-11-11T16:14:31,781 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:31,781 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:31,781 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:31,783 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:31,783 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:31,783 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:31,784 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/437ed8c719f7475d915e338456d73f54, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d55eb47e689e4d23a8023a875260c272, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cba89fc9eb05420d95c882197aba5bc8] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=36.9 K 2024-11-11T16:14:31,784 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437ed8c719f7475d915e338456d73f54, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341669548 2024-11-11T16:14:31,785 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55eb47e689e4d23a8023a875260c272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731341669779 2024-11-11T16:14:31,785 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba89fc9eb05420d95c882197aba5bc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731341670566 2024-11-11T16:14:31,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741924_1100 (size=12301) 2024-11-11T16:14:31,827 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:31,828 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/55ed0c351bf049ed9ad22d8f08281313 is 50, key is test_row_0/C:col10/1731341670953/Put/seqid=0 2024-11-11T16:14:31,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341731826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341731828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:31,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741925_1101 (size=13255) 2024-11-11T16:14:31,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:31,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341731937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341731937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T16:14:32,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341732157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341732159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,169 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/954d5abeabaf4d77ac65286e5ba9eaab as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/954d5abeabaf4d77ac65286e5ba9eaab 2024-11-11T16:14:32,183 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 954d5abeabaf4d77ac65286e5ba9eaab(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:32,183 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:32,183 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341671646; duration=0sec 2024-11-11T16:14:32,183 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:32,183 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:32,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731341732198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,204 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:32,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731341732206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,212 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,216 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f1e1099f6cbb40158f9a53e48c49b3c9 2024-11-11T16:14:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54392 deadline: 1731341732216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,229 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in af9dedd7afe35ed4a9eb53f0210465b1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:14:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d6944795588d4013906701c6aa3ae322 is 50, key is test_row_0/C:col10/1731341671127/Put/seqid=0 2024-11-11T16:14:32,267 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/55ed0c351bf049ed9ad22d8f08281313 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/55ed0c351bf049ed9ad22d8f08281313 2024-11-11T16:14:32,276 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into 55ed0c351bf049ed9ad22d8f08281313(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:32,276 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:32,276 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341671646; duration=0sec 2024-11-11T16:14:32,277 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:32,277 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:32,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741926_1102 (size=12301) 2024-11-11T16:14:32,311 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d6944795588d4013906701c6aa3ae322 2024-11-11T16:14:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b24fc372b6704f27949308b858aae1c8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b24fc372b6704f27949308b858aae1c8 2024-11-11T16:14:32,325 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b24fc372b6704f27949308b858aae1c8, entries=150, sequenceid=397, filesize=12.0 K 2024-11-11T16:14:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/f1e1099f6cbb40158f9a53e48c49b3c9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f1e1099f6cbb40158f9a53e48c49b3c9 2024-11-11T16:14:32,335 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f1e1099f6cbb40158f9a53e48c49b3c9, entries=150, sequenceid=397, filesize=12.0 K 2024-11-11T16:14:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d6944795588d4013906701c6aa3ae322 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d6944795588d4013906701c6aa3ae322 2024-11-11T16:14:32,343 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d6944795588d4013906701c6aa3ae322, entries=150, sequenceid=397, filesize=12.0 K 2024-11-11T16:14:32,345 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for af9dedd7afe35ed4a9eb53f0210465b1 in 684ms, sequenceid=397, compaction requested=false 2024-11-11T16:14:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:32,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T16:14:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T16:14:32,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T16:14:32,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4230 sec 2024-11-11T16:14:32,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.4340 sec 2024-11-11T16:14:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:32,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:14:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:32,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/a38cfc4e114b471183f9d5c220d3486a is 50, key is test_row_0/A:col10/1731341671816/Put/seqid=0 2024-11-11T16:14:32,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741927_1103 (size=12301) 2024-11-11T16:14:32,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/a38cfc4e114b471183f9d5c220d3486a 2024-11-11T16:14:32,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d1d286c36e6d41ad8d3e7c0f6fa9af90 is 50, key is test_row_0/B:col10/1731341671816/Put/seqid=0 2024-11-11T16:14:32,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341732532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341732534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741928_1104 (size=12301) 2024-11-11T16:14:32,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d1d286c36e6d41ad8d3e7c0f6fa9af90 2024-11-11T16:14:32,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/3269c72593c741929ebc2031bfe258ea is 50, key is test_row_0/C:col10/1731341671816/Put/seqid=0 2024-11-11T16:14:32,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741929_1105 (size=12301) 2024-11-11T16:14:32,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/3269c72593c741929ebc2031bfe258ea 2024-11-11T16:14:32,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/a38cfc4e114b471183f9d5c220d3486a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a38cfc4e114b471183f9d5c220d3486a 2024-11-11T16:14:32,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a38cfc4e114b471183f9d5c220d3486a, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T16:14:32,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d1d286c36e6d41ad8d3e7c0f6fa9af90 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d1d286c36e6d41ad8d3e7c0f6fa9af90 2024-11-11T16:14:32,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d1d286c36e6d41ad8d3e7c0f6fa9af90, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T16:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/3269c72593c741929ebc2031bfe258ea as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/3269c72593c741929ebc2031bfe258ea 2024-11-11T16:14:32,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/3269c72593c741929ebc2031bfe258ea, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T16:14:32,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for af9dedd7afe35ed4a9eb53f0210465b1 in 161ms, sequenceid=415, compaction requested=true 2024-11-11T16:14:32,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:32,641 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:32,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:32,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:32,642 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:32,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:32,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:32,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:32,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:32,643 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:32,643 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:32,643 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:32,644 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/954d5abeabaf4d77ac65286e5ba9eaab, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b24fc372b6704f27949308b858aae1c8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a38cfc4e114b471183f9d5c220d3486a] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=37.0 K 2024-11-11T16:14:32,644 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:32,644 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:32,644 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:32,644 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/829b21f5332a497bad2c0759f6d1f367, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f1e1099f6cbb40158f9a53e48c49b3c9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d1d286c36e6d41ad8d3e7c0f6fa9af90] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=37.0 K 2024-11-11T16:14:32,645 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954d5abeabaf4d77ac65286e5ba9eaab, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731341670566 2024-11-11T16:14:32,647 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 829b21f5332a497bad2c0759f6d1f367, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731341670566 2024-11-11T16:14:32,648 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24fc372b6704f27949308b858aae1c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731341671108 2024-11-11T16:14:32,648 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e1099f6cbb40158f9a53e48c49b3c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731341671108 2024-11-11T16:14:32,648 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38cfc4e114b471183f9d5c220d3486a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731341671816 2024-11-11T16:14:32,649 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d286c36e6d41ad8d3e7c0f6fa9af90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731341671816 2024-11-11T16:14:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:32,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T16:14:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:32,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:32,660 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:32,661 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:32,661 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f0acc1043d7f4c849d8d4c282a34cc39 is 50, key is test_row_0/A:col10/1731341671816/Put/seqid=0 2024-11-11T16:14:32,661 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/611fbd4b4c0a42bd96dee2ee3471ab5b is 50, key is test_row_0/B:col10/1731341671816/Put/seqid=0 2024-11-11T16:14:32,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994 is 50, key is test_row_0/A:col10/1731341672649/Put/seqid=0 2024-11-11T16:14:32,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741931_1107 (size=13357) 2024-11-11T16:14:32,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741930_1106 (size=13357) 2024-11-11T16:14:32,688 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/f0acc1043d7f4c849d8d4c282a34cc39 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f0acc1043d7f4c849d8d4c282a34cc39 2024-11-11T16:14:32,702 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/611fbd4b4c0a42bd96dee2ee3471ab5b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/611fbd4b4c0a42bd96dee2ee3471ab5b 2024-11-11T16:14:32,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741932_1108 (size=14741) 2024-11-11T16:14:32,707 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into f0acc1043d7f4c849d8d4c282a34cc39(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:32,707 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:32,707 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341672641; duration=0sec 2024-11-11T16:14:32,707 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:32,707 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:32,707 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:32,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994 2024-11-11T16:14:32,710 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:32,710 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:32,710 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:32,710 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/55ed0c351bf049ed9ad22d8f08281313, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d6944795588d4013906701c6aa3ae322, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/3269c72593c741929ebc2031bfe258ea] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=37.0 K 2024-11-11T16:14:32,712 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ed0c351bf049ed9ad22d8f08281313, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731341670566 2024-11-11T16:14:32,712 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into 611fbd4b4c0a42bd96dee2ee3471ab5b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:32,712 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:32,712 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341672641; duration=0sec 2024-11-11T16:14:32,712 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:32,712 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:32,712 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6944795588d4013906701c6aa3ae322, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731341671108 2024-11-11T16:14:32,713 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3269c72593c741929ebc2031bfe258ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731341671816 2024-11-11T16:14:32,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341732712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341732723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,732 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:32,733 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/ad58e5dad8704663b06b086aff902587 is 50, key is test_row_0/C:col10/1731341671816/Put/seqid=0 2024-11-11T16:14:32,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741933_1109 (size=13357) 2024-11-11T16:14:32,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/098af10eb3dd46a6a5dc09528cb42c7b is 50, key is test_row_0/B:col10/1731341672649/Put/seqid=0 2024-11-11T16:14:32,772 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/ad58e5dad8704663b06b086aff902587 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ad58e5dad8704663b06b086aff902587 2024-11-11T16:14:32,785 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into ad58e5dad8704663b06b086aff902587(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:32,785 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:32,785 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341672642; duration=0sec 2024-11-11T16:14:32,785 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:32,785 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:32,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741934_1110 (size=12301) 2024-11-11T16:14:32,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/098af10eb3dd46a6a5dc09528cb42c7b 2024-11-11T16:14:32,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d901ea23a9784cec8faa70720baeab51 is 50, key is test_row_0/C:col10/1731341672649/Put/seqid=0 2024-11-11T16:14:32,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341732826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341732858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:32,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741935_1111 (size=12301) 2024-11-11T16:14:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341733036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341733066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:33,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d901ea23a9784cec8faa70720baeab51 2024-11-11T16:14:33,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994 2024-11-11T16:14:33,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994, entries=200, sequenceid=434, filesize=14.4 K 2024-11-11T16:14:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/098af10eb3dd46a6a5dc09528cb42c7b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/098af10eb3dd46a6a5dc09528cb42c7b 2024-11-11T16:14:33,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/098af10eb3dd46a6a5dc09528cb42c7b, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T16:14:33,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d901ea23a9784cec8faa70720baeab51 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d901ea23a9784cec8faa70720baeab51 2024-11-11T16:14:33,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d901ea23a9784cec8faa70720baeab51, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T16:14:33,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for af9dedd7afe35ed4a9eb53f0210465b1 in 666ms, sequenceid=434, compaction requested=false 2024-11-11T16:14:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:33,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T16:14:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:33,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/9953bd1887b44f858572f15ed295f466 is 50, key is test_row_0/A:col10/1731341673359/Put/seqid=0 2024-11-11T16:14:33,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741936_1112 (size=12301) 2024-11-11T16:14:33,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/9953bd1887b44f858572f15ed295f466 2024-11-11T16:14:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341733430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341733426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:33,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/96f0319f8d6e44d99dc0ae1093312bae is 50, key is test_row_0/B:col10/1731341673359/Put/seqid=0 2024-11-11T16:14:33,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741937_1113 (size=12301) 2024-11-11T16:14:33,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/96f0319f8d6e44d99dc0ae1093312bae 2024-11-11T16:14:33,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/edcce0a5d73a42d7bbb1fcfadbaaae95 is 50, key is test_row_0/C:col10/1731341673359/Put/seqid=0 2024-11-11T16:14:33,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341733546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341733546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:33,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741938_1114 (size=12301) 2024-11-11T16:14:33,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341733759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341733760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/edcce0a5d73a42d7bbb1fcfadbaaae95 2024-11-11T16:14:34,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/9953bd1887b44f858572f15ed295f466 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/9953bd1887b44f858572f15ed295f466 2024-11-11T16:14:34,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/9953bd1887b44f858572f15ed295f466, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T16:14:34,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/96f0319f8d6e44d99dc0ae1093312bae as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/96f0319f8d6e44d99dc0ae1093312bae 2024-11-11T16:14:34,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/96f0319f8d6e44d99dc0ae1093312bae, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T16:14:34,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/edcce0a5d73a42d7bbb1fcfadbaaae95 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/edcce0a5d73a42d7bbb1fcfadbaaae95 2024-11-11T16:14:34,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/edcce0a5d73a42d7bbb1fcfadbaaae95, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T16:14:34,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for af9dedd7afe35ed4a9eb53f0210465b1 in 723ms, sequenceid=451, compaction requested=true 2024-11-11T16:14:34,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:34,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:34,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:34,083 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:34,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af9dedd7afe35ed4a9eb53f0210465b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:34,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:14:34,084 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:34,088 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:34,088 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/A is initiating minor compaction (all files) 2024-11-11T16:14:34,088 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/A in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,088 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f0acc1043d7f4c849d8d4c282a34cc39, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/9953bd1887b44f858572f15ed295f466] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=39.5 K 2024-11-11T16:14:34,089 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:34,089 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/B is initiating minor compaction (all files) 2024-11-11T16:14:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:34,089 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/B in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,089 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/611fbd4b4c0a42bd96dee2ee3471ab5b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/098af10eb3dd46a6a5dc09528cb42c7b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/96f0319f8d6e44d99dc0ae1093312bae] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=37.1 K 2024-11-11T16:14:34,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:14:34,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:34,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:34,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:34,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:34,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:34,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:34,092 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0acc1043d7f4c849d8d4c282a34cc39, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731341671816 2024-11-11T16:14:34,092 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 611fbd4b4c0a42bd96dee2ee3471ab5b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731341671816 2024-11-11T16:14:34,093 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5aeb2e3c67c4bb1b4aeef5c96ecb994, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731341672528 2024-11-11T16:14:34,093 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 098af10eb3dd46a6a5dc09528cb42c7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731341672528 2024-11-11T16:14:34,093 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9953bd1887b44f858572f15ed295f466, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731341672709 2024-11-11T16:14:34,094 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f0319f8d6e44d99dc0ae1093312bae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731341672709 2024-11-11T16:14:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T16:14:34,096 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:34,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T16:14:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T16:14:34,101 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:34,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/85cd23b8206a4f2c8a34dc50648a3983 is 50, key is test_row_0/A:col10/1731341674087/Put/seqid=0 2024-11-11T16:14:34,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:34,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:34,130 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:34,131 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/87d816372181438680bbde7332b80d36 is 50, key is test_row_0/B:col10/1731341673359/Put/seqid=0 2024-11-11T16:14:34,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741939_1115 (size=12301) 2024-11-11T16:14:34,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/85cd23b8206a4f2c8a34dc50648a3983 2024-11-11T16:14:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341734129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,138 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#A#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341734131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,139 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/6508dc9fc8604b3c934cde51e4f4de4a is 50, key is test_row_0/A:col10/1731341673359/Put/seqid=0 2024-11-11T16:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d2c97e1af6e246328612ce25a2d48527 is 50, key is test_row_0/B:col10/1731341674087/Put/seqid=0 2024-11-11T16:14:34,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741941_1117 (size=13459) 2024-11-11T16:14:34,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741940_1116 (size=13459) 2024-11-11T16:14:34,194 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/6508dc9fc8604b3c934cde51e4f4de4a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6508dc9fc8604b3c934cde51e4f4de4a 2024-11-11T16:14:34,197 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/87d816372181438680bbde7332b80d36 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/87d816372181438680bbde7332b80d36 2024-11-11T16:14:34,202 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/A of af9dedd7afe35ed4a9eb53f0210465b1 into 6508dc9fc8604b3c934cde51e4f4de4a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:34,202 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:34,202 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/A, priority=13, startTime=1731341674083; duration=0sec 2024-11-11T16:14:34,202 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:34,202 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:A 2024-11-11T16:14:34,202 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:34,205 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/B of af9dedd7afe35ed4a9eb53f0210465b1 into 87d816372181438680bbde7332b80d36(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:34,205 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:34,205 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/B, priority=13, startTime=1731341674083; duration=0sec 2024-11-11T16:14:34,205 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:34,205 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:B 2024-11-11T16:14:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T16:14:34,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:34,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): af9dedd7afe35ed4a9eb53f0210465b1/C is initiating minor compaction (all files) 2024-11-11T16:14:34,206 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af9dedd7afe35ed4a9eb53f0210465b1/C in TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,206 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ad58e5dad8704663b06b086aff902587, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d901ea23a9784cec8faa70720baeab51, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/edcce0a5d73a42d7bbb1fcfadbaaae95] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp, totalSize=37.1 K 2024-11-11T16:14:34,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad58e5dad8704663b06b086aff902587, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731341671816 2024-11-11T16:14:34,207 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d901ea23a9784cec8faa70720baeab51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731341672528 2024-11-11T16:14:34,207 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting edcce0a5d73a42d7bbb1fcfadbaaae95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731341672709 2024-11-11T16:14:34,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741942_1118 (size=12301) 2024-11-11T16:14:34,235 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af9dedd7afe35ed4a9eb53f0210465b1#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:34,236 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/306ee77cdfcb4fb08e5e058fd74ecccd is 50, key is test_row_0/C:col10/1731341673359/Put/seqid=0 2024-11-11T16:14:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341734246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341734246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T16:14:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741943_1119 (size=13459) 2024-11-11T16:14:34,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T16:14:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T16:14:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341734458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341734457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T16:14:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d2c97e1af6e246328612ce25a2d48527 2024-11-11T16:14:34,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d0261f417eac49daa70fbb33c17537e1 is 50, key is test_row_0/C:col10/1731341674087/Put/seqid=0 2024-11-11T16:14:34,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741944_1120 (size=12301) 2024-11-11T16:14:34,681 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/306ee77cdfcb4fb08e5e058fd74ecccd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/306ee77cdfcb4fb08e5e058fd74ecccd 2024-11-11T16:14:34,688 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af9dedd7afe35ed4a9eb53f0210465b1/C of af9dedd7afe35ed4a9eb53f0210465b1 into 306ee77cdfcb4fb08e5e058fd74ecccd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:34,688 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:34,688 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., storeName=af9dedd7afe35ed4a9eb53f0210465b1/C, priority=13, startTime=1731341674084; duration=0sec 2024-11-11T16:14:34,688 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:34,688 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af9dedd7afe35ed4a9eb53f0210465b1:C 2024-11-11T16:14:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T16:14:34,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T16:14:34,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:34,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731341734766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731341734766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:14:34,846 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T16:14:34,846 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:34,847 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T16:14:34,847 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:34,847 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T16:14:34,847 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:34,852 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T16:14:34,852 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:34,853 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,853 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,853 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,853 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,853 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,853 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,853 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:34,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:34,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:34,854 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:34,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:34,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T16:14:34,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:34,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:34,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:35,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T16:14:35,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:35,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:35,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:35,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:35,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:35,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d0261f417eac49daa70fbb33c17537e1 2024-11-11T16:14:35,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/85cd23b8206a4f2c8a34dc50648a3983 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/85cd23b8206a4f2c8a34dc50648a3983 2024-11-11T16:14:35,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/85cd23b8206a4f2c8a34dc50648a3983, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T16:14:35,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/d2c97e1af6e246328612ce25a2d48527 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d2c97e1af6e246328612ce25a2d48527 2024-11-11T16:14:35,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d2c97e1af6e246328612ce25a2d48527, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T16:14:35,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/d0261f417eac49daa70fbb33c17537e1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d0261f417eac49daa70fbb33c17537e1 2024-11-11T16:14:35,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d0261f417eac49daa70fbb33c17537e1, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T16:14:35,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for af9dedd7afe35ed4a9eb53f0210465b1 in 990ms, sequenceid=471, compaction requested=false 2024-11-11T16:14:35,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:35,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T16:14:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:35,194 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T16:14:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:35,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b140621a270f4fdc9f279127f9a35476 is 50, key is test_row_0/A:col10/1731341674114/Put/seqid=0 2024-11-11T16:14:35,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741945_1121 (size=12301) 2024-11-11T16:14:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T16:14:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:35,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. as already flushing 2024-11-11T16:14:35,278 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T16:14:35,278 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T16:14:35,278 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:35,278 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:35,278 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:35,278 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:35,278 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:35,278 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:35,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:35,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:35,604 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b140621a270f4fdc9f279127f9a35476 2024-11-11T16:14:35,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/6de6c00b12c44aa698d955d02781423b is 50, key is test_row_0/B:col10/1731341674114/Put/seqid=0 2024-11-11T16:14:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741946_1122 (size=12301) 2024-11-11T16:14:36,016 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/6de6c00b12c44aa698d955d02781423b 2024-11-11T16:14:36,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/656514e893f045d596b5bbe241310b91 is 50, key is test_row_0/C:col10/1731341674114/Put/seqid=0 2024-11-11T16:14:36,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741947_1123 (size=12301) 2024-11-11T16:14:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T16:14:36,361 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T16:14:36,429 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/656514e893f045d596b5bbe241310b91 2024-11-11T16:14:36,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/b140621a270f4fdc9f279127f9a35476 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b140621a270f4fdc9f279127f9a35476 2024-11-11T16:14:36,439 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b140621a270f4fdc9f279127f9a35476, entries=150, sequenceid=486, filesize=12.0 K 2024-11-11T16:14:36,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/6de6c00b12c44aa698d955d02781423b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6de6c00b12c44aa698d955d02781423b 2024-11-11T16:14:36,444 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6de6c00b12c44aa698d955d02781423b, entries=150, sequenceid=486, filesize=12.0 K 2024-11-11T16:14:36,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/656514e893f045d596b5bbe241310b91 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/656514e893f045d596b5bbe241310b91 2024-11-11T16:14:36,448 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/656514e893f045d596b5bbe241310b91, entries=150, sequenceid=486, filesize=12.0 K 2024-11-11T16:14:36,449 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for af9dedd7afe35ed4a9eb53f0210465b1 in 1255ms, sequenceid=486, compaction requested=true 2024-11-11T16:14:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for af9dedd7afe35ed4a9eb53f0210465b1: 2024-11-11T16:14:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T16:14:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T16:14:36,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T16:14:36,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3470 sec 2024-11-11T16:14:36,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.3570 sec 2024-11-11T16:14:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T16:14:38,276 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:42,268 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T16:14:42,268 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:42,268 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,268 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:42,278 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T16:14:42,278 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:42,278 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,278 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:42,288 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T16:14:42,288 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:14:42,288 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,288 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:42,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T16:14:42,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-11T16:14:42,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-11T16:14:42,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-11T16:14:42,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-11T16:14:42,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2194 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2099 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1020 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3058 rows 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1011 2024-11-11T16:14:42,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3032 rows 2024-11-11T16:14:42,289 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:14:42,289 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:14:42,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:42,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:14:42,295 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T16:14:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:42,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341682305"}]},"ts":"1731341682305"} 2024-11-11T16:14:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T16:14:42,307 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T16:14:42,307 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T16:14:42,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:14:42,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, UNASSIGN}] 2024-11-11T16:14:42,315 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, UNASSIGN 2024-11-11T16:14:42,317 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=af9dedd7afe35ed4a9eb53f0210465b1, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:42,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, UNASSIGN because future has completed 2024-11-11T16:14:42,332 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:14:42,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:14:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T16:14:42,490 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing af9dedd7afe35ed4a9eb53f0210465b1, disabling compactions & flushes 2024-11-11T16:14:42,491 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. after waiting 0 ms 2024-11-11T16:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:42,492 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing af9dedd7afe35ed4a9eb53f0210465b1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T16:14:42,492 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=A 2024-11-11T16:14:42,492 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:42,492 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=B 2024-11-11T16:14:42,492 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:42,492 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK af9dedd7afe35ed4a9eb53f0210465b1, store=C 2024-11-11T16:14:42,492 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/abe5ca0569a74ae5a4f35c75dd3e62a9 is 50, key is test_row_0/A:col10/1731341675276/Put/seqid=0 2024-11-11T16:14:42,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741948_1124 (size=12301) 2024-11-11T16:14:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T16:14:42,902 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/abe5ca0569a74ae5a4f35c75dd3e62a9 2024-11-11T16:14:42,911 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/8eb8aafbe8a24dc29400baff257ce1e5 is 50, key is test_row_0/B:col10/1731341675276/Put/seqid=0 2024-11-11T16:14:42,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741949_1125 (size=12301) 2024-11-11T16:14:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T16:14:43,315 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/8eb8aafbe8a24dc29400baff257ce1e5 2024-11-11T16:14:43,323 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/5f0e39a3e02d43ef9a62e092b139cdcc is 50, key is test_row_0/C:col10/1731341675276/Put/seqid=0 2024-11-11T16:14:43,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741950_1126 (size=12301) 2024-11-11T16:14:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T16:14:43,727 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/5f0e39a3e02d43ef9a62e092b139cdcc 2024-11-11T16:14:43,733 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/A/abe5ca0569a74ae5a4f35c75dd3e62a9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/abe5ca0569a74ae5a4f35c75dd3e62a9 2024-11-11T16:14:43,737 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/abe5ca0569a74ae5a4f35c75dd3e62a9, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T16:14:43,738 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/B/8eb8aafbe8a24dc29400baff257ce1e5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/8eb8aafbe8a24dc29400baff257ce1e5 2024-11-11T16:14:43,742 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/8eb8aafbe8a24dc29400baff257ce1e5, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T16:14:43,743 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/.tmp/C/5f0e39a3e02d43ef9a62e092b139cdcc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5f0e39a3e02d43ef9a62e092b139cdcc 2024-11-11T16:14:43,748 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5f0e39a3e02d43ef9a62e092b139cdcc, entries=150, sequenceid=494, filesize=12.0 K 2024-11-11T16:14:43,749 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for af9dedd7afe35ed4a9eb53f0210465b1 in 1257ms, sequenceid=494, compaction requested=true 2024-11-11T16:14:43,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/feaaf95e76024f96b22f4ee5d2d2ed49, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b9be62f619b14c3685d0eb63f06ef734, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/c6d71d062fce4477a7c236f3e2c5665b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/48489375dec74b01ad7ea92b8172776f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f4a8b1ad6ae540dc8c2917535cfc9096, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1e996c3eac184fda88ec0a57cab34fda, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e938230e4a0741ceadf7a5693bbc8dcd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a903e129ce1a4fb6bab46fe3545deec1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/4d7bed22c2b0435eb9388e681a80513a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/d90539be4a064526a99a1c036a62c0de, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cec164d9003142498f850f360bd68b34, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6c1072d91e184161867a09c540646cfd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/2561f149ecb9453f8234e07e9cb65eb9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6672c70f317545cd969af9d0db660365, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1d3c82780f0e4945ac52c0f55ad99de0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/23027db177a14d2eaae579bc5eb83b62, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f645e8d7951d4236b7641543f5d14066, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/52e8e706a45e48b6b120fc277a5e4d9c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/79b6edbc61354308a2023d57df5aad02, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/57a7b20a064b415bbbd84c6f9a7e686f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/66eb546ea3f7421f9753cacc3d90886c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/152a9a1364154ea582591bfe48f20808, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/be8e6fe71aeb46edb5aedd8bd7737243, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cfd7cc8e15154e89b0c0cb9a66c15451, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/14d2dc098def4c9fa040a7b84540d33e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/30d775fe287f4565b1ec41b7540032e0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f6ff48e2af424ab1a670095f5763e8f1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/eeaab9b4ef77415985ad739a1ae4af10, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/954d5abeabaf4d77ac65286e5ba9eaab, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b24fc372b6704f27949308b858aae1c8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f0acc1043d7f4c849d8d4c282a34cc39, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a38cfc4e114b471183f9d5c220d3486a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/9953bd1887b44f858572f15ed295f466] to archive 2024-11-11T16:14:43,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:14:43,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/feaaf95e76024f96b22f4ee5d2d2ed49 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/feaaf95e76024f96b22f4ee5d2d2ed49 2024-11-11T16:14:43,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b9be62f619b14c3685d0eb63f06ef734 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b9be62f619b14c3685d0eb63f06ef734 2024-11-11T16:14:43,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/c6d71d062fce4477a7c236f3e2c5665b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/c6d71d062fce4477a7c236f3e2c5665b 2024-11-11T16:14:43,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/48489375dec74b01ad7ea92b8172776f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/48489375dec74b01ad7ea92b8172776f 2024-11-11T16:14:43,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f4a8b1ad6ae540dc8c2917535cfc9096 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f4a8b1ad6ae540dc8c2917535cfc9096 2024-11-11T16:14:43,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1e996c3eac184fda88ec0a57cab34fda to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1e996c3eac184fda88ec0a57cab34fda 2024-11-11T16:14:43,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e938230e4a0741ceadf7a5693bbc8dcd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e938230e4a0741ceadf7a5693bbc8dcd 2024-11-11T16:14:43,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a903e129ce1a4fb6bab46fe3545deec1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a903e129ce1a4fb6bab46fe3545deec1 2024-11-11T16:14:43,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/4d7bed22c2b0435eb9388e681a80513a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/4d7bed22c2b0435eb9388e681a80513a 2024-11-11T16:14:43,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/d90539be4a064526a99a1c036a62c0de to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/d90539be4a064526a99a1c036a62c0de 2024-11-11T16:14:43,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cec164d9003142498f850f360bd68b34 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cec164d9003142498f850f360bd68b34 2024-11-11T16:14:43,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6c1072d91e184161867a09c540646cfd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6c1072d91e184161867a09c540646cfd 2024-11-11T16:14:43,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/2561f149ecb9453f8234e07e9cb65eb9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/2561f149ecb9453f8234e07e9cb65eb9 2024-11-11T16:14:43,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6672c70f317545cd969af9d0db660365 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6672c70f317545cd969af9d0db660365 2024-11-11T16:14:43,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1d3c82780f0e4945ac52c0f55ad99de0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/1d3c82780f0e4945ac52c0f55ad99de0 2024-11-11T16:14:43,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/23027db177a14d2eaae579bc5eb83b62 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/23027db177a14d2eaae579bc5eb83b62 2024-11-11T16:14:43,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f645e8d7951d4236b7641543f5d14066 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f645e8d7951d4236b7641543f5d14066 2024-11-11T16:14:43,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/52e8e706a45e48b6b120fc277a5e4d9c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/52e8e706a45e48b6b120fc277a5e4d9c 2024-11-11T16:14:43,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/79b6edbc61354308a2023d57df5aad02 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/79b6edbc61354308a2023d57df5aad02 2024-11-11T16:14:43,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/57a7b20a064b415bbbd84c6f9a7e686f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/57a7b20a064b415bbbd84c6f9a7e686f 2024-11-11T16:14:43,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/66eb546ea3f7421f9753cacc3d90886c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/66eb546ea3f7421f9753cacc3d90886c 2024-11-11T16:14:43,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/152a9a1364154ea582591bfe48f20808 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/152a9a1364154ea582591bfe48f20808 2024-11-11T16:14:43,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/be8e6fe71aeb46edb5aedd8bd7737243 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/be8e6fe71aeb46edb5aedd8bd7737243 2024-11-11T16:14:43,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cfd7cc8e15154e89b0c0cb9a66c15451 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/cfd7cc8e15154e89b0c0cb9a66c15451 2024-11-11T16:14:43,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/14d2dc098def4c9fa040a7b84540d33e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/14d2dc098def4c9fa040a7b84540d33e 2024-11-11T16:14:43,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/30d775fe287f4565b1ec41b7540032e0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/30d775fe287f4565b1ec41b7540032e0 2024-11-11T16:14:43,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f6ff48e2af424ab1a670095f5763e8f1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f6ff48e2af424ab1a670095f5763e8f1 2024-11-11T16:14:43,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/eeaab9b4ef77415985ad739a1ae4af10 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/eeaab9b4ef77415985ad739a1ae4af10 2024-11-11T16:14:43,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/954d5abeabaf4d77ac65286e5ba9eaab to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/954d5abeabaf4d77ac65286e5ba9eaab 2024-11-11T16:14:43,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b24fc372b6704f27949308b858aae1c8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b24fc372b6704f27949308b858aae1c8 2024-11-11T16:14:43,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f0acc1043d7f4c849d8d4c282a34cc39 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/f0acc1043d7f4c849d8d4c282a34cc39 2024-11-11T16:14:43,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a38cfc4e114b471183f9d5c220d3486a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/a38cfc4e114b471183f9d5c220d3486a 2024-11-11T16:14:43,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/e5aeb2e3c67c4bb1b4aeef5c96ecb994 2024-11-11T16:14:43,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/9953bd1887b44f858572f15ed295f466 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/9953bd1887b44f858572f15ed295f466 2024-11-11T16:14:43,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b3a5e1867c3d47f092948029771b8d44, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/29818c46503c421fbc8af32d2932731a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/5b912354d7f1497db6810e1104a1c270, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/9dc959f2f6074b36b123f21bafed94af, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/924f4c8382ab4f08844b7ac96e93d716, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b7f51055f172450f964b7b84faf5dbbb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b975ad3b681a42b3b3e13f18a8cc2687, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d3ac281fda314f8b8c03b15939357f3d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f05507cdb085418f8cb2d7ef6c6d33fe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/80d8c291021a4850ad582cf01891e377, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/3d85ca404c2b4580986d9581334450ca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7855259c3e9542b7be7e89ac0cbdf3f2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/ef560d922c81467eb8ee7cd5b6b05a25, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/53090d875acc4e73af992c7c23297d43, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/143dc9c5df3c46cfa57d33719346ab94, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/807d6d9db12344bcbbd350764c348f4b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/cc703283a4d247f2a812f79a305321ee, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/c198802167674de9a11ce2f30fa42c1c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/e57f5703b38948819b439d41a5092ae6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/09ca7d0968354d9b90a0e8f1c143f75e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f6ddcf6dd2d6484cbf517041a49d69b7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6e5d8a1e095f47e5b6a517d9a67bf95f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7668c133b53f4147b5ddac731d949c7d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/07febddf1c544495a4b57b9b59676cf7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/37425e2f3331446185008d7ab6e324aa, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/0a65cb6c732b415dab6735ea2f59f41a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/2349e9e1d72740f194f61572ce15d8ae, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/829b21f5332a497bad2c0759f6d1f367, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/024c0c2a703e4b379b03ce1bb1df568e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f1e1099f6cbb40158f9a53e48c49b3c9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/611fbd4b4c0a42bd96dee2ee3471ab5b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d1d286c36e6d41ad8d3e7c0f6fa9af90, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/098af10eb3dd46a6a5dc09528cb42c7b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/96f0319f8d6e44d99dc0ae1093312bae] to archive 2024-11-11T16:14:43,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:14:43,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b3a5e1867c3d47f092948029771b8d44 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b3a5e1867c3d47f092948029771b8d44 2024-11-11T16:14:43,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/29818c46503c421fbc8af32d2932731a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/29818c46503c421fbc8af32d2932731a 2024-11-11T16:14:43,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/5b912354d7f1497db6810e1104a1c270 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/5b912354d7f1497db6810e1104a1c270 2024-11-11T16:14:43,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/9dc959f2f6074b36b123f21bafed94af to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/9dc959f2f6074b36b123f21bafed94af 2024-11-11T16:14:43,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/924f4c8382ab4f08844b7ac96e93d716 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/924f4c8382ab4f08844b7ac96e93d716 2024-11-11T16:14:43,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b7f51055f172450f964b7b84faf5dbbb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b7f51055f172450f964b7b84faf5dbbb 2024-11-11T16:14:43,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b975ad3b681a42b3b3e13f18a8cc2687 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/b975ad3b681a42b3b3e13f18a8cc2687 2024-11-11T16:14:43,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d3ac281fda314f8b8c03b15939357f3d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d3ac281fda314f8b8c03b15939357f3d 2024-11-11T16:14:43,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f05507cdb085418f8cb2d7ef6c6d33fe to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f05507cdb085418f8cb2d7ef6c6d33fe 2024-11-11T16:14:43,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/80d8c291021a4850ad582cf01891e377 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/80d8c291021a4850ad582cf01891e377 2024-11-11T16:14:43,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/3d85ca404c2b4580986d9581334450ca to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/3d85ca404c2b4580986d9581334450ca 2024-11-11T16:14:43,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7855259c3e9542b7be7e89ac0cbdf3f2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7855259c3e9542b7be7e89ac0cbdf3f2 2024-11-11T16:14:43,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/ef560d922c81467eb8ee7cd5b6b05a25 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/ef560d922c81467eb8ee7cd5b6b05a25 2024-11-11T16:14:43,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/53090d875acc4e73af992c7c23297d43 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/53090d875acc4e73af992c7c23297d43 2024-11-11T16:14:43,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/143dc9c5df3c46cfa57d33719346ab94 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/143dc9c5df3c46cfa57d33719346ab94 2024-11-11T16:14:43,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/807d6d9db12344bcbbd350764c348f4b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/807d6d9db12344bcbbd350764c348f4b 2024-11-11T16:14:43,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/cc703283a4d247f2a812f79a305321ee to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/cc703283a4d247f2a812f79a305321ee 2024-11-11T16:14:43,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/c198802167674de9a11ce2f30fa42c1c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/c198802167674de9a11ce2f30fa42c1c 2024-11-11T16:14:43,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/e57f5703b38948819b439d41a5092ae6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/e57f5703b38948819b439d41a5092ae6 2024-11-11T16:14:43,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/09ca7d0968354d9b90a0e8f1c143f75e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/09ca7d0968354d9b90a0e8f1c143f75e 2024-11-11T16:14:43,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f6ddcf6dd2d6484cbf517041a49d69b7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f6ddcf6dd2d6484cbf517041a49d69b7 2024-11-11T16:14:43,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6e5d8a1e095f47e5b6a517d9a67bf95f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6e5d8a1e095f47e5b6a517d9a67bf95f 2024-11-11T16:14:43,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7668c133b53f4147b5ddac731d949c7d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/7668c133b53f4147b5ddac731d949c7d 2024-11-11T16:14:43,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/07febddf1c544495a4b57b9b59676cf7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/07febddf1c544495a4b57b9b59676cf7 2024-11-11T16:14:43,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/37425e2f3331446185008d7ab6e324aa to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/37425e2f3331446185008d7ab6e324aa 2024-11-11T16:14:43,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/0a65cb6c732b415dab6735ea2f59f41a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/0a65cb6c732b415dab6735ea2f59f41a 2024-11-11T16:14:43,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/2349e9e1d72740f194f61572ce15d8ae to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/2349e9e1d72740f194f61572ce15d8ae 2024-11-11T16:14:43,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/829b21f5332a497bad2c0759f6d1f367 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/829b21f5332a497bad2c0759f6d1f367 2024-11-11T16:14:43,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/024c0c2a703e4b379b03ce1bb1df568e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/024c0c2a703e4b379b03ce1bb1df568e 2024-11-11T16:14:43,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f1e1099f6cbb40158f9a53e48c49b3c9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/f1e1099f6cbb40158f9a53e48c49b3c9 2024-11-11T16:14:43,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/611fbd4b4c0a42bd96dee2ee3471ab5b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/611fbd4b4c0a42bd96dee2ee3471ab5b 2024-11-11T16:14:43,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d1d286c36e6d41ad8d3e7c0f6fa9af90 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d1d286c36e6d41ad8d3e7c0f6fa9af90 2024-11-11T16:14:43,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/098af10eb3dd46a6a5dc09528cb42c7b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/098af10eb3dd46a6a5dc09528cb42c7b 2024-11-11T16:14:43,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/96f0319f8d6e44d99dc0ae1093312bae to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/96f0319f8d6e44d99dc0ae1093312bae 2024-11-11T16:14:43,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ea00cea79f554709b306b02f3b24af46, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/1eba8ecc14f04a1db8f7ba3e2afe941b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/57f3a4e08c5241f59e0f80b593fe94a8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/0614c82fcf674d5b933ac061added1ec, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c7f1cb2a988b4eee86b3e1f3764084e6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a80a4fa2457440d19f78957eb9fe6ef0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c628215bc339485bab467eafb0832ab7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5500718ed30f40f99aa29c21f64c9b4d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/9871d226eced4f4293a1d8a58564b368, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/b053b0abba2d44fca0033ab6dca79a8d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/dbc051e1978142bbb258b2d941f6226d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/f6ba42aa934c4f25bceb89cc2a8a0785, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/40f6c5b95ce14f3480efaec56831c760, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/986025dcf23445e6bb46a59309a81a7c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/8d58164573e94fd39cf6c473e67f93ae, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cd9ed7b454ff48aa8e967f0c64554237, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/2bb1129a10be45bc82bb3eb435e7dce0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/39c3058cf32c41a59521eef9a8742062, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ff744e3cee6544ae96381df17ccb05c1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a0e9471e7ecb4f198a002c05cdd8b0ba, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/323743b7f8a0496ca35e5fba0a5982ce, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/375ab23f9b264075b39298afcb65358b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/80666943af17471c82521430d21b8a85, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a8e72f84ead74286931d28b8e62a6747, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/437ed8c719f7475d915e338456d73f54, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/df49c5819c79463eb750b63ca18e631d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d55eb47e689e4d23a8023a875260c272, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/55ed0c351bf049ed9ad22d8f08281313, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cba89fc9eb05420d95c882197aba5bc8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d6944795588d4013906701c6aa3ae322, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ad58e5dad8704663b06b086aff902587, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/3269c72593c741929ebc2031bfe258ea, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d901ea23a9784cec8faa70720baeab51, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/edcce0a5d73a42d7bbb1fcfadbaaae95] to archive 2024-11-11T16:14:43,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:14:43,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ea00cea79f554709b306b02f3b24af46 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ea00cea79f554709b306b02f3b24af46 2024-11-11T16:14:43,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/1eba8ecc14f04a1db8f7ba3e2afe941b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/1eba8ecc14f04a1db8f7ba3e2afe941b 2024-11-11T16:14:43,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/57f3a4e08c5241f59e0f80b593fe94a8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/57f3a4e08c5241f59e0f80b593fe94a8 2024-11-11T16:14:43,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/0614c82fcf674d5b933ac061added1ec to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/0614c82fcf674d5b933ac061added1ec 2024-11-11T16:14:43,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c7f1cb2a988b4eee86b3e1f3764084e6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c7f1cb2a988b4eee86b3e1f3764084e6 2024-11-11T16:14:43,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a80a4fa2457440d19f78957eb9fe6ef0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a80a4fa2457440d19f78957eb9fe6ef0 2024-11-11T16:14:43,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c628215bc339485bab467eafb0832ab7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/c628215bc339485bab467eafb0832ab7 2024-11-11T16:14:43,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5500718ed30f40f99aa29c21f64c9b4d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5500718ed30f40f99aa29c21f64c9b4d 2024-11-11T16:14:43,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/9871d226eced4f4293a1d8a58564b368 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/9871d226eced4f4293a1d8a58564b368 2024-11-11T16:14:43,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/b053b0abba2d44fca0033ab6dca79a8d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/b053b0abba2d44fca0033ab6dca79a8d 2024-11-11T16:14:43,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/dbc051e1978142bbb258b2d941f6226d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/dbc051e1978142bbb258b2d941f6226d 2024-11-11T16:14:43,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/f6ba42aa934c4f25bceb89cc2a8a0785 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/f6ba42aa934c4f25bceb89cc2a8a0785 2024-11-11T16:14:43,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/40f6c5b95ce14f3480efaec56831c760 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/40f6c5b95ce14f3480efaec56831c760 2024-11-11T16:14:43,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/986025dcf23445e6bb46a59309a81a7c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/986025dcf23445e6bb46a59309a81a7c 2024-11-11T16:14:43,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/8d58164573e94fd39cf6c473e67f93ae to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/8d58164573e94fd39cf6c473e67f93ae 2024-11-11T16:14:43,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cd9ed7b454ff48aa8e967f0c64554237 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cd9ed7b454ff48aa8e967f0c64554237 2024-11-11T16:14:43,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/2bb1129a10be45bc82bb3eb435e7dce0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/2bb1129a10be45bc82bb3eb435e7dce0 2024-11-11T16:14:43,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/39c3058cf32c41a59521eef9a8742062 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/39c3058cf32c41a59521eef9a8742062 2024-11-11T16:14:43,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ff744e3cee6544ae96381df17ccb05c1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ff744e3cee6544ae96381df17ccb05c1 2024-11-11T16:14:43,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a0e9471e7ecb4f198a002c05cdd8b0ba to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a0e9471e7ecb4f198a002c05cdd8b0ba 2024-11-11T16:14:43,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/323743b7f8a0496ca35e5fba0a5982ce to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/323743b7f8a0496ca35e5fba0a5982ce 2024-11-11T16:14:43,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/375ab23f9b264075b39298afcb65358b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/375ab23f9b264075b39298afcb65358b 2024-11-11T16:14:43,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/80666943af17471c82521430d21b8a85 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/80666943af17471c82521430d21b8a85 2024-11-11T16:14:43,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a8e72f84ead74286931d28b8e62a6747 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/a8e72f84ead74286931d28b8e62a6747 2024-11-11T16:14:43,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/437ed8c719f7475d915e338456d73f54 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/437ed8c719f7475d915e338456d73f54 2024-11-11T16:14:43,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/df49c5819c79463eb750b63ca18e631d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/df49c5819c79463eb750b63ca18e631d 2024-11-11T16:14:43,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d55eb47e689e4d23a8023a875260c272 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d55eb47e689e4d23a8023a875260c272 2024-11-11T16:14:43,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/55ed0c351bf049ed9ad22d8f08281313 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/55ed0c351bf049ed9ad22d8f08281313 2024-11-11T16:14:43,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cba89fc9eb05420d95c882197aba5bc8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/cba89fc9eb05420d95c882197aba5bc8 2024-11-11T16:14:43,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d6944795588d4013906701c6aa3ae322 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d6944795588d4013906701c6aa3ae322 2024-11-11T16:14:43,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ad58e5dad8704663b06b086aff902587 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/ad58e5dad8704663b06b086aff902587 2024-11-11T16:14:43,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/3269c72593c741929ebc2031bfe258ea to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/3269c72593c741929ebc2031bfe258ea 2024-11-11T16:14:43,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d901ea23a9784cec8faa70720baeab51 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d901ea23a9784cec8faa70720baeab51 2024-11-11T16:14:43,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/edcce0a5d73a42d7bbb1fcfadbaaae95 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/edcce0a5d73a42d7bbb1fcfadbaaae95 2024-11-11T16:14:43,907 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=1 2024-11-11T16:14:43,908 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. 2024-11-11T16:14:43,908 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for af9dedd7afe35ed4a9eb53f0210465b1: Waiting for close lock at 1731341682491Running coprocessor pre-close hooks at 1731341682491Disabling compacts and flushes for region at 1731341682491Disabling writes for close at 1731341682491Obtaining lock to block concurrent updates at 1731341682492 (+1 ms)Preparing flush snapshotting stores in af9dedd7afe35ed4a9eb53f0210465b1 at 1731341682492Finished memstore snapshotting TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731341682492Flushing stores of TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1. at 1731341682493 (+1 ms)Flushing af9dedd7afe35ed4a9eb53f0210465b1/A: creating writer at 1731341682493Flushing af9dedd7afe35ed4a9eb53f0210465b1/A: appending metadata at 1731341682496 (+3 ms)Flushing af9dedd7afe35ed4a9eb53f0210465b1/A: closing flushed file at 1731341682496Flushing af9dedd7afe35ed4a9eb53f0210465b1/B: creating writer at 1731341682907 (+411 ms)Flushing af9dedd7afe35ed4a9eb53f0210465b1/B: appending metadata at 1731341682910 (+3 ms)Flushing af9dedd7afe35ed4a9eb53f0210465b1/B: closing flushed file at 1731341682910Flushing af9dedd7afe35ed4a9eb53f0210465b1/C: creating writer at 1731341683320 (+410 ms)Flushing af9dedd7afe35ed4a9eb53f0210465b1/C: appending metadata at 1731341683323 (+3 ms)Flushing af9dedd7afe35ed4a9eb53f0210465b1/C: closing flushed file at 1731341683323Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55ed9fda: reopening flushed file at 1731341683732 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ea977f8: reopening flushed file at 1731341683738 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b0732f7: reopening flushed file at 1731341683743 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for af9dedd7afe35ed4a9eb53f0210465b1 in 1257ms, sequenceid=494, compaction requested=true at 1731341683749 (+6 ms)Writing region close event to WAL at 1731341683903 (+154 ms)Running coprocessor post-close hooks at 1731341683907 (+4 ms)Closed at 1731341683908 (+1 ms) 2024-11-11T16:14:43,910 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:43,911 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=af9dedd7afe35ed4a9eb53f0210465b1, regionState=CLOSED 2024-11-11T16:14:43,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:14:43,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T16:14:43,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure af9dedd7afe35ed4a9eb53f0210465b1, server=16b413a53992,45121,1731341648557 in 1.5810 sec 2024-11-11T16:14:43,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-11T16:14:43,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af9dedd7afe35ed4a9eb53f0210465b1, UNASSIGN in 1.6010 sec 2024-11-11T16:14:43,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T16:14:43,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6090 sec 2024-11-11T16:14:43,920 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341683920"}]},"ts":"1731341683920"} 2024-11-11T16:14:43,921 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T16:14:43,922 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T16:14:43,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6250 sec 2024-11-11T16:14:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T16:14:44,456 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:44,460 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T16:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:44,465 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T16:14:44,467 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:44,472 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:44,475 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/recovered.edits] 2024-11-11T16:14:44,478 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6508dc9fc8604b3c934cde51e4f4de4a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/6508dc9fc8604b3c934cde51e4f4de4a 2024-11-11T16:14:44,479 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/85cd23b8206a4f2c8a34dc50648a3983 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/85cd23b8206a4f2c8a34dc50648a3983 2024-11-11T16:14:44,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/abe5ca0569a74ae5a4f35c75dd3e62a9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/abe5ca0569a74ae5a4f35c75dd3e62a9 2024-11-11T16:14:44,482 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b140621a270f4fdc9f279127f9a35476 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/A/b140621a270f4fdc9f279127f9a35476 2024-11-11T16:14:44,486 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6de6c00b12c44aa698d955d02781423b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/6de6c00b12c44aa698d955d02781423b 2024-11-11T16:14:44,488 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/87d816372181438680bbde7332b80d36 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/87d816372181438680bbde7332b80d36 2024-11-11T16:14:44,489 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/8eb8aafbe8a24dc29400baff257ce1e5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/8eb8aafbe8a24dc29400baff257ce1e5 2024-11-11T16:14:44,491 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d2c97e1af6e246328612ce25a2d48527 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/B/d2c97e1af6e246328612ce25a2d48527 2024-11-11T16:14:44,494 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/306ee77cdfcb4fb08e5e058fd74ecccd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/306ee77cdfcb4fb08e5e058fd74ecccd 2024-11-11T16:14:44,495 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5f0e39a3e02d43ef9a62e092b139cdcc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/5f0e39a3e02d43ef9a62e092b139cdcc 2024-11-11T16:14:44,497 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/656514e893f045d596b5bbe241310b91 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/656514e893f045d596b5bbe241310b91 2024-11-11T16:14:44,498 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d0261f417eac49daa70fbb33c17537e1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/C/d0261f417eac49daa70fbb33c17537e1 2024-11-11T16:14:44,502 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/recovered.edits/497.seqid to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1/recovered.edits/497.seqid 2024-11-11T16:14:44,502 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/af9dedd7afe35ed4a9eb53f0210465b1 2024-11-11T16:14:44,502 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T16:14:44,508 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:44,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T16:14:44,520 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T16:14:44,557 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T16:14:44,559 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:44,559 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T16:14:44,559 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731341684559"}]},"ts":"9223372036854775807"} 2024-11-11T16:14:44,565 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T16:14:44,565 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => af9dedd7afe35ed4a9eb53f0210465b1, NAME => 'TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T16:14:44,565 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T16:14:44,566 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731341684566"}]},"ts":"9223372036854775807"} 2024-11-11T16:14:44,569 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T16:14:44,570 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:44,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 110 msec 2024-11-11T16:14:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T16:14:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T16:14:44,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:44,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=243 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/16b413a53992:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;16b413a53992:45121-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1554041885_22 at /127.0.0.1:39386 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/16b413a53992:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=651 (was 544) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2467 (was 2614) 2024-11-11T16:14:44,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=243, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=651, ProcessCount=11, AvailableMemoryMB=2467 2024-11-11T16:14:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:14:44,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:14:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:44,603 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T16:14:44,603 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:44,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-11T16:14:44,604 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T16:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T16:14:44,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741951_1127 (size=1061) 2024-11-11T16:14:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T16:14:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T16:14:45,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:14:45,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741952_1128 (size=53) 2024-11-11T16:14:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T16:14:45,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:45,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c44034eee8380292fc203bc6a539d00c, disabling compactions & flushes 2024-11-11T16:14:45,421 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. after waiting 0 ms 2024-11-11T16:14:45,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,421 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c44034eee8380292fc203bc6a539d00c: Waiting for close lock at 1731341685420Disabling compacts and flushes for region at 1731341685420Disabling writes for close at 1731341685421 (+1 ms)Writing region close event to WAL at 1731341685421Closed at 1731341685421 2024-11-11T16:14:45,422 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T16:14:45,422 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731341685422"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341685422"}]},"ts":"1731341685422"} 2024-11-11T16:14:45,424 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T16:14:45,425 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T16:14:45,425 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341685425"}]},"ts":"1731341685425"} 2024-11-11T16:14:45,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T16:14:45,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, ASSIGN}] 2024-11-11T16:14:45,428 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, ASSIGN 2024-11-11T16:14:45,429 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, ASSIGN; state=OFFLINE, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=false 2024-11-11T16:14:45,580 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:45,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, ASSIGN because future has completed 2024-11-11T16:14:45,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:14:45,739 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,739 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:14:45,739 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,739 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:45,739 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,740 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,741 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,742 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:45,743 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c44034eee8380292fc203bc6a539d00c columnFamilyName A 2024-11-11T16:14:45,743 DEBUG [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:45,744 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(327): Store=c44034eee8380292fc203bc6a539d00c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:45,744 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,745 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:45,745 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c44034eee8380292fc203bc6a539d00c columnFamilyName B 2024-11-11T16:14:45,745 DEBUG [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:45,745 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(327): Store=c44034eee8380292fc203bc6a539d00c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T16:14:45,746 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,746 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:45,747 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c44034eee8380292fc203bc6a539d00c columnFamilyName C 2024-11-11T16:14:45,747 DEBUG [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:45,747 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(327): Store=c44034eee8380292fc203bc6a539d00c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:45,747 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,747 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,748 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,748 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,749 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,749 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,750 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:14:45,751 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,752 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:14:45,753 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened c44034eee8380292fc203bc6a539d00c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63719876, jitterRate=-0.05049985647201538}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:14:45,753 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:45,754 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for c44034eee8380292fc203bc6a539d00c: Running coprocessor pre-open hook at 1731341685740Writing region info on filesystem at 1731341685740Initializing all the Stores at 1731341685740Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341685740Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341685741 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341685741Cleaning up temporary data from old regions at 1731341685749 (+8 ms)Running coprocessor post-open hooks at 1731341685753 (+4 ms)Region opened successfully at 1731341685754 (+1 ms) 2024-11-11T16:14:45,754 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., pid=32, masterSystemTime=1731341685735 2024-11-11T16:14:45,756 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,756 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:45,757 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:45,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:14:45,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T16:14:45,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 in 176 msec 2024-11-11T16:14:45,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-11T16:14:45,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, ASSIGN in 334 msec 2024-11-11T16:14:45,763 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T16:14:45,763 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341685763"}]},"ts":"1731341685763"} 2024-11-11T16:14:45,765 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T16:14:45,765 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T16:14:45,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-11T16:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T16:14:46,756 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:46,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@704cd315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:46,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:46,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:46,759 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce15b3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:46,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aafdf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:46,763 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:46,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:46,767 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:14:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,33601,1731341647570 2024-11-11T16:14:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c1b1db5 2024-11-11T16:14:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:14:46,769 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:14:46,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:14:46,775 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:14:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T16:14:46,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741953_1129 (size=1097) 2024-11-11T16:14:47,197 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T16:14:47,198 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T16:14:47,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:14:47,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, REOPEN/MOVE}] 2024-11-11T16:14:47,209 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, REOPEN/MOVE 2024-11-11T16:14:47,209 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, REOPEN/MOVE because future has completed 2024-11-11T16:14:47,211 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:14:47,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:14:47,364 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing c44034eee8380292fc203bc6a539d00c, disabling compactions & flushes 2024-11-11T16:14:47,364 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. after waiting 0 ms 2024-11-11T16:14:47,364 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,368 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:14:47,369 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,369 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for c44034eee8380292fc203bc6a539d00c: Waiting for close lock at 1731341687364Running coprocessor pre-close hooks at 1731341687364Disabling compacts and flushes for region at 1731341687364Disabling writes for close at 1731341687364Writing region close event to WAL at 1731341687365 (+1 ms)Running coprocessor post-close hooks at 1731341687369 (+4 ms)Closed at 1731341687369 2024-11-11T16:14:47,369 WARN [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: c44034eee8380292fc203bc6a539d00c to self. 2024-11-11T16:14:47,371 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,372 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=CLOSED 2024-11-11T16:14:47,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:14:47,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T16:14:47,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 in 164 msec 2024-11-11T16:14:47,377 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, REOPEN/MOVE; state=CLOSED, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=true 2024-11-11T16:14:47,528 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, REOPEN/MOVE because future has completed 2024-11-11T16:14:47,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:14:47,686 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,686 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:14:47,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:14:47,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,690 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,690 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:47,696 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c44034eee8380292fc203bc6a539d00c columnFamilyName A 2024-11-11T16:14:47,698 DEBUG [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:47,698 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(327): Store=c44034eee8380292fc203bc6a539d00c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:47,699 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,700 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:47,700 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c44034eee8380292fc203bc6a539d00c columnFamilyName B 2024-11-11T16:14:47,700 DEBUG [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:47,700 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(327): Store=c44034eee8380292fc203bc6a539d00c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:47,700 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,701 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:14:47,701 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c44034eee8380292fc203bc6a539d00c columnFamilyName C 2024-11-11T16:14:47,701 DEBUG [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:47,702 INFO [StoreOpener-c44034eee8380292fc203bc6a539d00c-1 {}] regionserver.HStore(327): Store=c44034eee8380292fc203bc6a539d00c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:14:47,702 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,702 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,703 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,704 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,705 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,705 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,706 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:14:47,707 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,708 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c44034eee8380292fc203bc6a539d00c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59768978, jitterRate=-0.1093728244304657}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:14:47,708 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,710 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c44034eee8380292fc203bc6a539d00c: Running coprocessor pre-open hook at 1731341687687Writing region info on filesystem at 1731341687687Initializing all the Stores at 1731341687688 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341687688Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341687688Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341687689 (+1 ms)Cleaning up temporary data from old regions at 1731341687705 (+16 ms)Running coprocessor post-open hooks at 1731341687708 (+3 ms)Region opened successfully at 1731341687710 (+2 ms) 2024-11-11T16:14:47,711 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., pid=37, masterSystemTime=1731341687682 2024-11-11T16:14:47,713 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,714 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,714 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=OPEN, openSeqNum=5, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:14:47,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-11T16:14:47,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 in 187 msec 2024-11-11T16:14:47,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T16:14:47,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, REOPEN/MOVE in 517 msec 2024-11-11T16:14:47,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T16:14:47,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-11T16:14:47,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-11-11T16:14:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T16:14:47,737 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:47,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53284553, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,739 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7073cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,740 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd7ed2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331b2320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,743 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,744 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27dfd755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d645a45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,747 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4646942e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5239767b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec657f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6473d92d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d20100f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b516bff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,758 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646855ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,760 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aff82eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f80fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,762 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c23adfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,764 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6775a220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d89e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e721d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2441aae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc0fdf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,770 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a9f068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,772 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1266e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6078660f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:14:47,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:14:47,774 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e3ab6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:14:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:14:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:14:47,776 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:14:47,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30213712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:14:47,777 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,777 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,777 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,778 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,779 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-11T16:14:47,790 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T16:14:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,793 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:47,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,794 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:47,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:47,798 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,800 DEBUG [Thread-654 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,800 DEBUG [Thread-654 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,805 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,806 DEBUG [Thread-663 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,806 DEBUG [Thread-663 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:14:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,823 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:14:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:14:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:14:47,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:14:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:14:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T16:14:47,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116f1b6e84c0db4e54800df39f2ec8ab45_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341687810/Put/seqid=0 2024-11-11T16:14:47,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341747878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341747880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:47,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341747890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:47,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341747891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:47,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341747899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:47,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741954_1130 (size=17034) 2024-11-11T16:14:47,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:47,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116f1b6e84c0db4e54800df39f2ec8ab45_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116f1b6e84c0db4e54800df39f2ec8ab45_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:47,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/21c717d94543407fa994b03a4688cc83, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:47,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T16:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:47,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:47,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:47,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/21c717d94543407fa994b03a4688cc83 is 175, key is test_row_0/A:col10/1731341687810/Put/seqid=0 2024-11-11T16:14:48,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341748008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741955_1131 (size=48139) 2024-11-11T16:14:48,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/21c717d94543407fa994b03a4688cc83 2024-11-11T16:14:48,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341748012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341748012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341748017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341748026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/16abc07bf9e54275860e0371d78f73b6 is 50, key is test_row_0/B:col10/1731341687810/Put/seqid=0 2024-11-11T16:14:48,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T16:14:48,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:48,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:48,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:48,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:48,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T16:14:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:48,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741956_1132 (size=12001) 2024-11-11T16:14:48,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/16abc07bf9e54275860e0371d78f73b6 2024-11-11T16:14:48,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/edd954bd4b7545368415b997577302f4 is 50, key is test_row_0/C:col10/1731341687810/Put/seqid=0 2024-11-11T16:14:48,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741957_1133 (size=12001) 2024-11-11T16:14:48,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/edd954bd4b7545368415b997577302f4 2024-11-11T16:14:48,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/21c717d94543407fa994b03a4688cc83 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/21c717d94543407fa994b03a4688cc83 2024-11-11T16:14:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341748216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/21c717d94543407fa994b03a4688cc83, entries=250, sequenceid=16, filesize=47.0 K 2024-11-11T16:14:48,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/16abc07bf9e54275860e0371d78f73b6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/16abc07bf9e54275860e0371d78f73b6 2024-11-11T16:14:48,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341748236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/16abc07bf9e54275860e0371d78f73b6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T16:14:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341748236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341748246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341748246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/edd954bd4b7545368415b997577302f4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/edd954bd4b7545368415b997577302f4 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/edd954bd4b7545368415b997577302f4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T16:14:48,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T16:14:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:48,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c44034eee8380292fc203bc6a539d00c in 450ms, sequenceid=16, compaction requested=false 2024-11-11T16:14:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:48,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:48,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T16:14:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T16:14:48,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:48,417 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a07cc95e266248bf82a005ea9de537fa_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341687879/Put/seqid=0 2024-11-11T16:14:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741958_1134 (size=12154) 2024-11-11T16:14:48,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341748579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341748579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341748580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341748581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341748589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341748696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341748696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341748697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341748697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341748707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341748917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341748917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T16:14:48,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341748918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341748920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,932 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a07cc95e266248bf82a005ea9de537fa_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a07cc95e266248bf82a005ea9de537fa_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5648459dd3cc4b499e7beaf8d63a18d3, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5648459dd3cc4b499e7beaf8d63a18d3 is 175, key is test_row_0/A:col10/1731341687879/Put/seqid=0 2024-11-11T16:14:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341748920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:48,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741959_1135 (size=30955) 2024-11-11T16:14:48,958 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5648459dd3cc4b499e7beaf8d63a18d3 2024-11-11T16:14:48,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/e92ef4d8b6c04e77b3cdfe3a0ec02313 is 50, key is test_row_0/B:col10/1731341687879/Put/seqid=0 2024-11-11T16:14:49,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741960_1136 (size=12001) 2024-11-11T16:14:49,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341749237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341749238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341749239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341749239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341749246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,453 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/e92ef4d8b6c04e77b3cdfe3a0ec02313 2024-11-11T16:14:49,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/03fa96aea33a4e18a2fd540512c5499c is 50, key is test_row_0/C:col10/1731341687879/Put/seqid=0 2024-11-11T16:14:49,499 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T16:14:49,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741961_1137 (size=12001) 2024-11-11T16:14:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341749766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341749767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341749767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341749767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341749767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:49,920 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/03fa96aea33a4e18a2fd540512c5499c 2024-11-11T16:14:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5648459dd3cc4b499e7beaf8d63a18d3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5648459dd3cc4b499e7beaf8d63a18d3 2024-11-11T16:14:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T16:14:49,942 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5648459dd3cc4b499e7beaf8d63a18d3, entries=150, sequenceid=36, filesize=30.2 K 2024-11-11T16:14:49,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/e92ef4d8b6c04e77b3cdfe3a0ec02313 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e92ef4d8b6c04e77b3cdfe3a0ec02313 2024-11-11T16:14:49,949 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e92ef4d8b6c04e77b3cdfe3a0ec02313, entries=150, sequenceid=36, filesize=11.7 K 2024-11-11T16:14:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/03fa96aea33a4e18a2fd540512c5499c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/03fa96aea33a4e18a2fd540512c5499c 2024-11-11T16:14:49,961 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/03fa96aea33a4e18a2fd540512c5499c, entries=150, sequenceid=36, filesize=11.7 K 2024-11-11T16:14:49,962 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c44034eee8380292fc203bc6a539d00c in 1545ms, sequenceid=36, compaction requested=false 2024-11-11T16:14:49,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:49,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:49,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-11T16:14:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-11T16:14:49,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T16:14:49,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1710 sec 2024-11-11T16:14:49,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.1880 sec 2024-11-11T16:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:50,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:14:50,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:50,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111be14bfcd09ae40d39a6d11164f86646d_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341690780/Put/seqid=0 2024-11-11T16:14:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741962_1138 (size=14594) 2024-11-11T16:14:50,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341750820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341750826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341750828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341750830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341750832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:50,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111be14bfcd09ae40d39a6d11164f86646d_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be14bfcd09ae40d39a6d11164f86646d_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:50,859 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a6d78dba99e94ab6b10275ea2d6fc224, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:50,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a6d78dba99e94ab6b10275ea2d6fc224 is 175, key is test_row_0/A:col10/1731341690780/Put/seqid=0 2024-11-11T16:14:50,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741963_1139 (size=39549) 2024-11-11T16:14:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341750936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341750936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341750946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341750946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341750958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341751146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341751146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341751155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341751156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341751176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,292 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a6d78dba99e94ab6b10275ea2d6fc224 2024-11-11T16:14:51,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/4f339a55d9b0455abd85d337524952dd is 50, key is test_row_0/B:col10/1731341690780/Put/seqid=0 2024-11-11T16:14:51,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741964_1140 (size=12001) 2024-11-11T16:14:51,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341751466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341751466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341751466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341751475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341751496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/4f339a55d9b0455abd85d337524952dd 2024-11-11T16:14:51,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a95a69a9aaf54b9f9dad9fd5fe0fc897 is 50, key is test_row_0/C:col10/1731341690780/Put/seqid=0 2024-11-11T16:14:51,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741965_1141 (size=12001) 2024-11-11T16:14:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T16:14:51,947 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:51,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T16:14:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T16:14:51,950 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:51,950 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:51,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:51,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341751976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341751977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341751977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:51,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341751985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341752006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T16:14:52,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T16:14:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:52,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:52,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:52,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a95a69a9aaf54b9f9dad9fd5fe0fc897 2024-11-11T16:14:52,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a6d78dba99e94ab6b10275ea2d6fc224 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a6d78dba99e94ab6b10275ea2d6fc224 2024-11-11T16:14:52,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a6d78dba99e94ab6b10275ea2d6fc224, entries=200, sequenceid=49, filesize=38.6 K 2024-11-11T16:14:52,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/4f339a55d9b0455abd85d337524952dd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/4f339a55d9b0455abd85d337524952dd 2024-11-11T16:14:52,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/4f339a55d9b0455abd85d337524952dd, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T16:14:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a95a69a9aaf54b9f9dad9fd5fe0fc897 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a95a69a9aaf54b9f9dad9fd5fe0fc897 2024-11-11T16:14:52,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a95a69a9aaf54b9f9dad9fd5fe0fc897, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T16:14:52,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c44034eee8380292fc203bc6a539d00c in 1463ms, sequenceid=49, compaction requested=true 2024-11-11T16:14:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:52,248 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:52,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:52,249 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:52,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:52,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:52,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:52,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:52,249 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:52,249 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/A is initiating minor compaction (all files) 2024-11-11T16:14:52,250 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/A in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,250 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/21c717d94543407fa994b03a4688cc83, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5648459dd3cc4b499e7beaf8d63a18d3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a6d78dba99e94ab6b10275ea2d6fc224] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=115.9 K 2024-11-11T16:14:52,250 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,250 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/21c717d94543407fa994b03a4688cc83, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5648459dd3cc4b499e7beaf8d63a18d3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a6d78dba99e94ab6b10275ea2d6fc224] 2024-11-11T16:14:52,252 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:52,252 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c717d94543407fa994b03a4688cc83, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341687799 2024-11-11T16:14:52,252 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/B is initiating minor compaction (all files) 2024-11-11T16:14:52,252 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/B in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,252 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/16abc07bf9e54275860e0371d78f73b6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e92ef4d8b6c04e77b3cdfe3a0ec02313, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/4f339a55d9b0455abd85d337524952dd] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.2 K 2024-11-11T16:14:52,252 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5648459dd3cc4b499e7beaf8d63a18d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731341687877 2024-11-11T16:14:52,253 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 16abc07bf9e54275860e0371d78f73b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341687807 2024-11-11T16:14:52,253 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d78dba99e94ab6b10275ea2d6fc224, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731341688544 2024-11-11T16:14:52,254 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e92ef4d8b6c04e77b3cdfe3a0ec02313, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731341687877 2024-11-11T16:14:52,256 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f339a55d9b0455abd85d337524952dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731341688544 2024-11-11T16:14:52,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T16:14:52,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,260 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T16:14:52,277 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:52,291 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:52,292 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/fd5f993973fa4748a693cca2d608228a is 50, key is test_row_0/B:col10/1731341690780/Put/seqid=0 2024-11-11T16:14:52,293 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111aa86ace956aa4fbdae52314cf4b1529e_c44034eee8380292fc203bc6a539d00c store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:52,298 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111aa86ace956aa4fbdae52314cf4b1529e_c44034eee8380292fc203bc6a539d00c, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:52,298 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aa86ace956aa4fbdae52314cf4b1529e_c44034eee8380292fc203bc6a539d00c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112ce8dc44299c4eacafc0aef08c992303_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341690810/Put/seqid=0 2024-11-11T16:14:52,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741966_1142 (size=12104) 2024-11-11T16:14:52,351 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/fd5f993973fa4748a693cca2d608228a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fd5f993973fa4748a693cca2d608228a 2024-11-11T16:14:52,358 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/B of c44034eee8380292fc203bc6a539d00c into fd5f993973fa4748a693cca2d608228a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:52,358 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:52,358 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/B, priority=13, startTime=1731341692249; duration=0sec 2024-11-11T16:14:52,359 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:52,359 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:B 2024-11-11T16:14:52,359 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:52,360 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:52,361 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/C is initiating minor compaction (all files) 2024-11-11T16:14:52,361 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/C in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,361 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/edd954bd4b7545368415b997577302f4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/03fa96aea33a4e18a2fd540512c5499c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a95a69a9aaf54b9f9dad9fd5fe0fc897] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.2 K 2024-11-11T16:14:52,361 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting edd954bd4b7545368415b997577302f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341687807 2024-11-11T16:14:52,362 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 03fa96aea33a4e18a2fd540512c5499c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731341687877 2024-11-11T16:14:52,362 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a95a69a9aaf54b9f9dad9fd5fe0fc897, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731341688544 2024-11-11T16:14:52,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741967_1143 (size=4469) 2024-11-11T16:14:52,382 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#A#compaction#123 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:52,383 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/6d3d1f1b9ef141e68ca0c7a392f5454b is 175, key is test_row_0/A:col10/1731341690780/Put/seqid=0 2024-11-11T16:14:52,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741968_1144 (size=12154) 2024-11-11T16:14:52,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,403 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:52,408 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112ce8dc44299c4eacafc0aef08c992303_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112ce8dc44299c4eacafc0aef08c992303_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:52,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/db966dbbd879478985f2eb941a4be113, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:52,410 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/ac6290ce478240ebb6f3335dd73332e6 is 50, key is test_row_0/C:col10/1731341690780/Put/seqid=0 2024-11-11T16:14:52,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/db966dbbd879478985f2eb941a4be113 is 175, key is test_row_0/A:col10/1731341690810/Put/seqid=0 2024-11-11T16:14:52,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741969_1145 (size=31058) 2024-11-11T16:14:52,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741971_1147 (size=30955) 2024-11-11T16:14:52,476 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/db966dbbd879478985f2eb941a4be113 2024-11-11T16:14:52,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741970_1146 (size=12104) 2024-11-11T16:14:52,493 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/ac6290ce478240ebb6f3335dd73332e6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ac6290ce478240ebb6f3335dd73332e6 2024-11-11T16:14:52,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/eb8a245e2ae74f90bd28309a2f5f0508 is 50, key is test_row_0/B:col10/1731341690810/Put/seqid=0 2024-11-11T16:14:52,503 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/C of c44034eee8380292fc203bc6a539d00c into ac6290ce478240ebb6f3335dd73332e6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:52,503 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:52,503 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/C, priority=13, startTime=1731341692249; duration=0sec 2024-11-11T16:14:52,503 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:52,503 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:C 2024-11-11T16:14:52,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741972_1148 (size=12001) 2024-11-11T16:14:52,520 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/eb8a245e2ae74f90bd28309a2f5f0508 2024-11-11T16:14:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/b30f7b4d84344a1db4b141277af32fb1 is 50, key is test_row_0/C:col10/1731341690810/Put/seqid=0 2024-11-11T16:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T16:14:52,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741973_1149 (size=12001) 2024-11-11T16:14:52,603 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/b30f7b4d84344a1db4b141277af32fb1 2024-11-11T16:14:52,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/db966dbbd879478985f2eb941a4be113 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/db966dbbd879478985f2eb941a4be113 2024-11-11T16:14:52,619 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/db966dbbd879478985f2eb941a4be113, entries=150, sequenceid=68, filesize=30.2 K 2024-11-11T16:14:52,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/eb8a245e2ae74f90bd28309a2f5f0508 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/eb8a245e2ae74f90bd28309a2f5f0508 2024-11-11T16:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,631 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/eb8a245e2ae74f90bd28309a2f5f0508, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T16:14:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/b30f7b4d84344a1db4b141277af32fb1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/b30f7b4d84344a1db4b141277af32fb1 2024-11-11T16:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,644 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/b30f7b4d84344a1db4b141277af32fb1, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T16:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,645 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for c44034eee8380292fc203bc6a539d00c in 386ms, sequenceid=68, compaction requested=false 2024-11-11T16:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T16:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T16:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T16:14:52,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 696 msec 2024-11-11T16:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 704 msec 2024-11-11T16:14:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,866 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/6d3d1f1b9ef141e68ca0c7a392f5454b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6d3d1f1b9ef141e68ca0c7a392f5454b 2024-11-11T16:14:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,872 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/A of c44034eee8380292fc203bc6a539d00c into 6d3d1f1b9ef141e68ca0c7a392f5454b(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:52,872 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:52,873 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/A, priority=13, startTime=1731341692248; duration=0sec 2024-11-11T16:14:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,873 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:52,873 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:A 2024-11-11T16:14:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:14:53,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:53,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:53,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:53,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:53,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T16:14:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,093 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T16:14:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,098 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T16:14:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,101 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:53,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f468bb4d88554ef99d4307dbea7bdd04_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341693025/Put/seqid=0 2024-11-11T16:14:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741974_1150 (size=21918) 2024-11-11T16:14:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341693151, max timestamp=1731341693168Num uniques 859; 2024-11-11T16:14:53,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:14:53,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341693151, max timestamp=1731341693168Num uniques 859; 2024-11-11T16:14:53,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:14:53,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341693151, max timestamp=1731341693168Num uniques 859; 2024-11-11T16:14:53,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T16:14:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T16:14:53,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341753211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341753213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341753215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341753217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341753228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T16:14:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341753337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341753337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341753337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341753347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341753347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,356 INFO [master/16b413a53992:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T16:14:53,356 INFO [master/16b413a53992:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T16:14:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T16:14:53,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T16:14:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,521 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:53,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f468bb4d88554ef99d4307dbea7bdd04_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f468bb4d88554ef99d4307dbea7bdd04_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:53,528 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/0e61309a0aa64764b7d2db84e5df2338, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:53,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/0e61309a0aa64764b7d2db84e5df2338 is 175, key is test_row_0/A:col10/1731341693025/Put/seqid=0 2024-11-11T16:14:53,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741975_1151 (size=65323) 2024-11-11T16:14:53,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341753556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341753556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341753556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341753556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341753556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T16:14:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T16:14:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T16:14:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341753866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341753866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341753866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341753866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341753867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:53,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:53,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T16:14:53,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:53,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:53,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:53,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/0e61309a0aa64764b7d2db84e5df2338 2024-11-11T16:14:53,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/93287a51ffdf4bbebf814bed4fb37a52 is 50, key is test_row_0/B:col10/1731341693025/Put/seqid=0 2024-11-11T16:14:53,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741976_1152 (size=12001) 2024-11-11T16:14:53,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/93287a51ffdf4bbebf814bed4fb37a52 2024-11-11T16:14:53,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/0509b79519194e85953da9ecb295e99d is 50, key is test_row_0/C:col10/1731341693025/Put/seqid=0 2024-11-11T16:14:54,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741977_1153 (size=12001) 2024-11-11T16:14:54,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/0509b79519194e85953da9ecb295e99d 2024-11-11T16:14:54,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/0e61309a0aa64764b7d2db84e5df2338 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/0e61309a0aa64764b7d2db84e5df2338 2024-11-11T16:14:54,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T16:14:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:54,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:54,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/0e61309a0aa64764b7d2db84e5df2338, entries=350, sequenceid=81, filesize=63.8 K 2024-11-11T16:14:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/93287a51ffdf4bbebf814bed4fb37a52 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/93287a51ffdf4bbebf814bed4fb37a52 2024-11-11T16:14:54,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/93287a51ffdf4bbebf814bed4fb37a52, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T16:14:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/0509b79519194e85953da9ecb295e99d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0509b79519194e85953da9ecb295e99d 2024-11-11T16:14:54,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0509b79519194e85953da9ecb295e99d, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T16:14:54,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for c44034eee8380292fc203bc6a539d00c in 1005ms, sequenceid=81, compaction requested=true 2024-11-11T16:14:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:54,082 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:54,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:54,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:54,083 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:54,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:54,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:54,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:54,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:54,088 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:54,088 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/A is initiating minor compaction (all files) 2024-11-11T16:14:54,088 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/A in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:54,089 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6d3d1f1b9ef141e68ca0c7a392f5454b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/db966dbbd879478985f2eb941a4be113, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/0e61309a0aa64764b7d2db84e5df2338] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=124.4 K 2024-11-11T16:14:54,089 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:54,089 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6d3d1f1b9ef141e68ca0c7a392f5454b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/db966dbbd879478985f2eb941a4be113, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/0e61309a0aa64764b7d2db84e5df2338] 2024-11-11T16:14:54,092 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:54,092 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/B is initiating minor compaction (all files) 2024-11-11T16:14:54,092 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/B in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:54,092 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fd5f993973fa4748a693cca2d608228a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/eb8a245e2ae74f90bd28309a2f5f0508, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/93287a51ffdf4bbebf814bed4fb37a52] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.3 K 2024-11-11T16:14:54,092 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3d1f1b9ef141e68ca0c7a392f5454b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731341688544 2024-11-11T16:14:54,092 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5f993973fa4748a693cca2d608228a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731341688544 2024-11-11T16:14:54,093 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting db966dbbd879478985f2eb941a4be113, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731341690810 2024-11-11T16:14:54,094 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8a245e2ae74f90bd28309a2f5f0508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731341690810 2024-11-11T16:14:54,094 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e61309a0aa64764b7d2db84e5df2338, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731341693017 2024-11-11T16:14:54,097 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 93287a51ffdf4bbebf814bed4fb37a52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731341693017 2024-11-11T16:14:54,119 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:54,119 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/c8821e6f600c48989cb184c8e7756f21 is 50, key is test_row_0/B:col10/1731341693025/Put/seqid=0 2024-11-11T16:14:54,121 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:54,137 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411116ef69dbf760f46fa89352ce5de800228_c44034eee8380292fc203bc6a539d00c store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:54,140 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411116ef69dbf760f46fa89352ce5de800228_c44034eee8380292fc203bc6a539d00c, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:54,140 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ef69dbf760f46fa89352ce5de800228_c44034eee8380292fc203bc6a539d00c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:54,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741979_1155 (size=4469) 2024-11-11T16:14:54,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741978_1154 (size=12207) 2024-11-11T16:14:54,157 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#A#compaction#133 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:54,158 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/c3fe09b7f24749c38c45b95674bc2d6e is 175, key is test_row_0/A:col10/1731341693025/Put/seqid=0 2024-11-11T16:14:54,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741980_1156 (size=31161) 2024-11-11T16:14:54,179 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/c3fe09b7f24749c38c45b95674bc2d6e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c3fe09b7f24749c38c45b95674bc2d6e 2024-11-11T16:14:54,186 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/A of c44034eee8380292fc203bc6a539d00c into c3fe09b7f24749c38c45b95674bc2d6e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:54,186 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:54,186 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/A, priority=13, startTime=1731341694082; duration=0sec 2024-11-11T16:14:54,186 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:54,186 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:A 2024-11-11T16:14:54,186 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:54,187 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:54,187 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/C is initiating minor compaction (all files) 2024-11-11T16:14:54,187 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/C in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:54,188 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ac6290ce478240ebb6f3335dd73332e6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/b30f7b4d84344a1db4b141277af32fb1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0509b79519194e85953da9ecb295e99d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.3 K 2024-11-11T16:14:54,192 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac6290ce478240ebb6f3335dd73332e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731341688544 2024-11-11T16:14:54,193 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b30f7b4d84344a1db4b141277af32fb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731341690810 2024-11-11T16:14:54,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T16:14:54,194 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0509b79519194e85953da9ecb295e99d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731341693017 2024-11-11T16:14:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:54,194 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-11T16:14:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:54,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:54,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:54,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:14:54,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113b6ef6c0bcc44972b0c463b9f043eac1_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341693214/Put/seqid=0 2024-11-11T16:14:54,223 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:54,224 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/1c87b0e18aca498798bba26a2b141a11 is 50, key is test_row_0/C:col10/1731341693025/Put/seqid=0 2024-11-11T16:14:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T16:14:54,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741981_1157 (size=12154) 2024-11-11T16:14:54,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741982_1158 (size=12207) 2024-11-11T16:14:54,289 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/1c87b0e18aca498798bba26a2b141a11 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/1c87b0e18aca498798bba26a2b141a11 2024-11-11T16:14:54,298 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/C of c44034eee8380292fc203bc6a539d00c into 1c87b0e18aca498798bba26a2b141a11(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:54,298 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:54,298 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/C, priority=13, startTime=1731341694084; duration=0sec 2024-11-11T16:14:54,298 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:54,298 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:C 2024-11-11T16:14:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341754390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341754393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341754397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341754397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341754400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341754507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341754507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341754516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341754516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341754516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,573 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/c8821e6f600c48989cb184c8e7756f21 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/c8821e6f600c48989cb184c8e7756f21 2024-11-11T16:14:54,625 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/B of c44034eee8380292fc203bc6a539d00c into c8821e6f600c48989cb184c8e7756f21(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:54,625 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:54,626 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/B, priority=13, startTime=1731341694083; duration=0sec 2024-11-11T16:14:54,626 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:54,626 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:B 2024-11-11T16:14:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:54,667 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113b6ef6c0bcc44972b0c463b9f043eac1_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b6ef6c0bcc44972b0c463b9f043eac1_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/22779a5068b94128a4f82a8a019eca11, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/22779a5068b94128a4f82a8a019eca11 is 175, key is test_row_0/A:col10/1731341693214/Put/seqid=0 2024-11-11T16:14:54,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741983_1159 (size=30955) 2024-11-11T16:14:54,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341754717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341754717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341754729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341754736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341754736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341755036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341755036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341755056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341755056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341755056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,087 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/22779a5068b94128a4f82a8a019eca11 2024-11-11T16:14:55,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/ad6c1d31c60e484b9cd5c98808b34516 is 50, key is test_row_0/B:col10/1731341693214/Put/seqid=0 2024-11-11T16:14:55,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741984_1160 (size=12001) 2024-11-11T16:14:55,198 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/ad6c1d31c60e484b9cd5c98808b34516 2024-11-11T16:14:55,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/992806e1f8f24ff8a9af739c77557ee0 is 50, key is test_row_0/C:col10/1731341693214/Put/seqid=0 2024-11-11T16:14:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T16:14:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741985_1161 (size=12001) 2024-11-11T16:14:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341755556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341755556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341755577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341755577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341755596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:55,671 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/992806e1f8f24ff8a9af739c77557ee0 2024-11-11T16:14:55,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/22779a5068b94128a4f82a8a019eca11 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/22779a5068b94128a4f82a8a019eca11 2024-11-11T16:14:55,691 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/22779a5068b94128a4f82a8a019eca11, entries=150, sequenceid=119, filesize=30.2 K 2024-11-11T16:14:55,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/ad6c1d31c60e484b9cd5c98808b34516 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ad6c1d31c60e484b9cd5c98808b34516 2024-11-11T16:14:55,700 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ad6c1d31c60e484b9cd5c98808b34516, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T16:14:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/992806e1f8f24ff8a9af739c77557ee0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/992806e1f8f24ff8a9af739c77557ee0 2024-11-11T16:14:55,710 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/992806e1f8f24ff8a9af739c77557ee0, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T16:14:55,713 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for c44034eee8380292fc203bc6a539d00c in 1519ms, sequenceid=119, compaction requested=false 2024-11-11T16:14:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T16:14:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T16:14:55,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T16:14:55,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6140 sec 2024-11-11T16:14:55,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6270 sec 2024-11-11T16:14:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:56,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:14:56,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:56,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:56,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:56,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:56,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:56,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:56,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f8e3329572314d6dbf65b1a90a363743_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341696588/Put/seqid=0 2024-11-11T16:14:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341756633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341756635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741986_1162 (size=12304) 2024-11-11T16:14:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341756642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341756644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341756645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341756763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341756763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341756769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341756769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341756770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341756976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341756976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341756976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341756976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:56,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341756985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:57,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f8e3329572314d6dbf65b1a90a363743_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8e3329572314d6dbf65b1a90a363743_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:57,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/4d9ba34c09544fae8790ebc4cb39dfeb, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:57,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/4d9ba34c09544fae8790ebc4cb39dfeb is 175, key is test_row_0/A:col10/1731341696588/Put/seqid=0 2024-11-11T16:14:57,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741987_1163 (size=31105) 2024-11-11T16:14:57,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/4d9ba34c09544fae8790ebc4cb39dfeb 2024-11-11T16:14:57,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/246d9935e50145149fe828698a8dcb13 is 50, key is test_row_0/B:col10/1731341696588/Put/seqid=0 2024-11-11T16:14:57,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741988_1164 (size=12151) 2024-11-11T16:14:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T16:14:57,269 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:57,270 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T16:14:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T16:14:57,272 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:57,276 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:57,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:57,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341757286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341757286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341757289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341757289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341757296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T16:14:57,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T16:14:57,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:57,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/246d9935e50145149fe828698a8dcb13 2024-11-11T16:14:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T16:14:57,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T16:14:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/d3f9bbb6d122478187325895684c7efb is 50, key is test_row_0/C:col10/1731341696588/Put/seqid=0 2024-11-11T16:14:57,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741989_1165 (size=12151) 2024-11-11T16:14:57,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T16:14:57,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:57,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341757806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341757806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341757806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341757806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341757809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T16:14:57,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T16:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:57,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:58,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T16:14:58,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:58,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:58,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:58,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:58,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:58,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/d3f9bbb6d122478187325895684c7efb 2024-11-11T16:14:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:58,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/4d9ba34c09544fae8790ebc4cb39dfeb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/4d9ba34c09544fae8790ebc4cb39dfeb 2024-11-11T16:14:58,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/4d9ba34c09544fae8790ebc4cb39dfeb, entries=150, sequenceid=134, filesize=30.4 K 2024-11-11T16:14:58,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/246d9935e50145149fe828698a8dcb13 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/246d9935e50145149fe828698a8dcb13 2024-11-11T16:14:58,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/246d9935e50145149fe828698a8dcb13, entries=150, sequenceid=134, filesize=11.9 K 2024-11-11T16:14:58,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/d3f9bbb6d122478187325895684c7efb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d3f9bbb6d122478187325895684c7efb 2024-11-11T16:14:58,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d3f9bbb6d122478187325895684c7efb, entries=150, sequenceid=134, filesize=11.9 K 2024-11-11T16:14:58,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c44034eee8380292fc203bc6a539d00c in 1505ms, sequenceid=134, compaction requested=true 2024-11-11T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:58,103 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:14:58,103 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:58,105 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:58,105 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/B is initiating minor compaction (all files) 2024-11-11T16:14:58,105 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/B in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:58,105 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/c8821e6f600c48989cb184c8e7756f21, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ad6c1d31c60e484b9cd5c98808b34516, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/246d9935e50145149fe828698a8dcb13] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.5 K 2024-11-11T16:14:58,105 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:58,105 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/A is initiating minor compaction (all files) 2024-11-11T16:14:58,105 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/A in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:58,105 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c3fe09b7f24749c38c45b95674bc2d6e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/22779a5068b94128a4f82a8a019eca11, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/4d9ba34c09544fae8790ebc4cb39dfeb] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=91.0 K 2024-11-11T16:14:58,105 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:58,106 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c3fe09b7f24749c38c45b95674bc2d6e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/22779a5068b94128a4f82a8a019eca11, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/4d9ba34c09544fae8790ebc4cb39dfeb] 2024-11-11T16:14:58,106 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c8821e6f600c48989cb184c8e7756f21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731341693017 2024-11-11T16:14:58,106 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3fe09b7f24749c38c45b95674bc2d6e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731341693017 2024-11-11T16:14:58,106 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ad6c1d31c60e484b9cd5c98808b34516, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731341693212 2024-11-11T16:14:58,107 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22779a5068b94128a4f82a8a019eca11, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731341693212 2024-11-11T16:14:58,107 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 246d9935e50145149fe828698a8dcb13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341696588 2024-11-11T16:14:58,107 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d9ba34c09544fae8790ebc4cb39dfeb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341696588 2024-11-11T16:14:58,130 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:58,131 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/9735ef6fa40344af800a16153ab87f13 is 50, key is test_row_0/B:col10/1731341696588/Put/seqid=0 2024-11-11T16:14:58,132 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:58,148 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111180f401f8c62948a7a24b92dc4c5bd16c_c44034eee8380292fc203bc6a539d00c store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:58,151 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111180f401f8c62948a7a24b92dc4c5bd16c_c44034eee8380292fc203bc6a539d00c, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:58,151 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111180f401f8c62948a7a24b92dc4c5bd16c_c44034eee8380292fc203bc6a539d00c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:58,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741990_1166 (size=12459) 2024-11-11T16:14:58,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741991_1167 (size=4469) 2024-11-11T16:14:58,180 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/9735ef6fa40344af800a16153ab87f13 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9735ef6fa40344af800a16153ab87f13 2024-11-11T16:14:58,192 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/B of c44034eee8380292fc203bc6a539d00c into 9735ef6fa40344af800a16153ab87f13(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:58,193 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:58,193 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/B, priority=13, startTime=1731341698103; duration=0sec 2024-11-11T16:14:58,193 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:14:58,196 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:B 2024-11-11T16:14:58,196 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:14:58,198 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:14:58,199 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/C is initiating minor compaction (all files) 2024-11-11T16:14:58,200 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/C in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:58,200 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/1c87b0e18aca498798bba26a2b141a11, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/992806e1f8f24ff8a9af739c77557ee0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d3f9bbb6d122478187325895684c7efb] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.5 K 2024-11-11T16:14:58,202 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c87b0e18aca498798bba26a2b141a11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731341693017 2024-11-11T16:14:58,205 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 992806e1f8f24ff8a9af739c77557ee0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731341693212 2024-11-11T16:14:58,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d3f9bbb6d122478187325895684c7efb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341696588 2024-11-11T16:14:58,222 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:58,222 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/0b113f82d3514d8a8a01463fe2d170bb is 50, key is test_row_0/C:col10/1731341696588/Put/seqid=0 2024-11-11T16:14:58,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T16:14:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:58,224 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:14:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:58,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741992_1168 (size=12459) 2024-11-11T16:14:58,246 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/0b113f82d3514d8a8a01463fe2d170bb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0b113f82d3514d8a8a01463fe2d170bb 2024-11-11T16:14:58,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111169a86ded5adf46f5ab9210a311b33f23_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341696641/Put/seqid=0 2024-11-11T16:14:58,253 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/C of c44034eee8380292fc203bc6a539d00c into 0b113f82d3514d8a8a01463fe2d170bb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:58,253 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:58,253 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/C, priority=13, startTime=1731341698103; duration=0sec 2024-11-11T16:14:58,253 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:58,253 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:C 2024-11-11T16:14:58,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741993_1169 (size=12304) 2024-11-11T16:14:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T16:14:58,573 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:14:58,574 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/ef0c40fd60314fba88a61e3f4debfbd8 is 175, key is test_row_0/A:col10/1731341696588/Put/seqid=0 2024-11-11T16:14:58,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741994_1170 (size=31413) 2024-11-11T16:14:58,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:58,673 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111169a86ded5adf46f5ab9210a311b33f23_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169a86ded5adf46f5ab9210a311b33f23_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/af01e07fa0d245f6818b261982841c14, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:58,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/af01e07fa0d245f6818b261982841c14 is 175, key is test_row_0/A:col10/1731341696641/Put/seqid=0 2024-11-11T16:14:58,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741995_1171 (size=31105) 2024-11-11T16:14:58,684 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/af01e07fa0d245f6818b261982841c14 2024-11-11T16:14:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/0e5bd3f87caf4cacbe91bad5ddbc962e is 50, key is test_row_0/B:col10/1731341696641/Put/seqid=0 2024-11-11T16:14:58,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741996_1172 (size=12151) 2024-11-11T16:14:58,721 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/0e5bd3f87caf4cacbe91bad5ddbc962e 2024-11-11T16:14:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/730a5e21a2724dc28b267ebd2d2a3e3c is 50, key is test_row_0/C:col10/1731341696641/Put/seqid=0 2024-11-11T16:14:58,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741997_1173 (size=12151) 2024-11-11T16:14:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:58,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:58,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341758830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341758830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341758831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341758834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341758836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341758946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341758946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341758947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341758947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341758947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,001 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/ef0c40fd60314fba88a61e3f4debfbd8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/ef0c40fd60314fba88a61e3f4debfbd8 2024-11-11T16:14:59,014 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/A of c44034eee8380292fc203bc6a539d00c into ef0c40fd60314fba88a61e3f4debfbd8(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:14:59,015 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:59,015 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/A, priority=13, startTime=1731341698103; duration=0sec 2024-11-11T16:14:59,015 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:14:59,015 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:A 2024-11-11T16:14:59,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341759156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341759156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341759157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341759156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341759157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,188 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/730a5e21a2724dc28b267ebd2d2a3e3c 2024-11-11T16:14:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/af01e07fa0d245f6818b261982841c14 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/af01e07fa0d245f6818b261982841c14 2024-11-11T16:14:59,209 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/af01e07fa0d245f6818b261982841c14, entries=150, sequenceid=154, filesize=30.4 K 2024-11-11T16:14:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/0e5bd3f87caf4cacbe91bad5ddbc962e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/0e5bd3f87caf4cacbe91bad5ddbc962e 2024-11-11T16:14:59,226 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/0e5bd3f87caf4cacbe91bad5ddbc962e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T16:14:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/730a5e21a2724dc28b267ebd2d2a3e3c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/730a5e21a2724dc28b267ebd2d2a3e3c 2024-11-11T16:14:59,239 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/730a5e21a2724dc28b267ebd2d2a3e3c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T16:14:59,241 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c44034eee8380292fc203bc6a539d00c in 1017ms, sequenceid=154, compaction requested=false 2024-11-11T16:14:59,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:14:59,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:59,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T16:14:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T16:14:59,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T16:14:59,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9710 sec 2024-11-11T16:14:59,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9820 sec 2024-11-11T16:14:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T16:14:59,416 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:14:59,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:14:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T16:14:59,419 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:14:59,421 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:14:59,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:14:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T16:14:59,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T16:14:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:14:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:14:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:14:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:14:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:59,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f75d39c2da9a4116a7a4d79160bb46b7_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341698827/Put/seqid=0 2024-11-11T16:14:59,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341759497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341759499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341759499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341759502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341759504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T16:14:59,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741998_1174 (size=12304) 2024-11-11T16:14:59,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:14:59,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:59,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:59,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:59,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341759617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341759617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341759617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341759617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341759617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:14:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:59,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:59,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T16:14:59,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341759826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341759826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341759827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341759827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:14:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341759827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:14:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:14:59,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:14:59,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:59,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:14:59,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:14:59,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:14:59,929 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:14:59,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f75d39c2da9a4116a7a4d79160bb46b7_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f75d39c2da9a4116a7a4d79160bb46b7_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:14:59,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a1acaa34a5b84646933b065f4df477c8, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:14:59,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a1acaa34a5b84646933b065f4df477c8 is 175, key is test_row_0/A:col10/1731341698827/Put/seqid=0 2024-11-11T16:14:59,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741999_1175 (size=31105) 2024-11-11T16:14:59,977 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a1acaa34a5b84646933b065f4df477c8 2024-11-11T16:14:59,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/9ab6e899462c435face6bed486505fc7 is 50, key is test_row_0/B:col10/1731341698827/Put/seqid=0 2024-11-11T16:15:00,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742000_1176 (size=12151) 2024-11-11T16:15:00,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T16:15:00,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341760136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341760136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341760137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341760146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341760146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:00,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/9ab6e899462c435face6bed486505fc7 2024-11-11T16:15:00,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/69c8faa69e11441f8ac2776ceaaaa158 is 50, key is test_row_0/C:col10/1731341698827/Put/seqid=0 2024-11-11T16:15:00,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742001_1177 (size=12151) 2024-11-11T16:15:00,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/69c8faa69e11441f8ac2776ceaaaa158 2024-11-11T16:15:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/a1acaa34a5b84646933b065f4df477c8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a1acaa34a5b84646933b065f4df477c8 2024-11-11T16:15:00,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a1acaa34a5b84646933b065f4df477c8, entries=150, sequenceid=170, filesize=30.4 K 2024-11-11T16:15:00,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/9ab6e899462c435face6bed486505fc7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9ab6e899462c435face6bed486505fc7 2024-11-11T16:15:00,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9ab6e899462c435face6bed486505fc7, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T16:15:00,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/69c8faa69e11441f8ac2776ceaaaa158 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/69c8faa69e11441f8ac2776ceaaaa158 2024-11-11T16:15:00,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/69c8faa69e11441f8ac2776ceaaaa158, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T16:15:00,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c44034eee8380292fc203bc6a539d00c in 1080ms, sequenceid=170, compaction requested=true 2024-11-11T16:15:00,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:00,550 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:00,551 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T16:15:00,566 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:00,566 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/B is initiating minor compaction (all files) 2024-11-11T16:15:00,566 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/B in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,566 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9735ef6fa40344af800a16153ab87f13, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/0e5bd3f87caf4cacbe91bad5ddbc962e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9ab6e899462c435face6bed486505fc7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.9 K 2024-11-11T16:15:00,567 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:00,567 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9735ef6fa40344af800a16153ab87f13, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341696588 2024-11-11T16:15:00,567 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/A is initiating minor compaction (all files) 2024-11-11T16:15:00,567 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/A in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,567 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/ef0c40fd60314fba88a61e3f4debfbd8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/af01e07fa0d245f6818b261982841c14, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a1acaa34a5b84646933b065f4df477c8] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=91.4 K 2024-11-11T16:15:00,567 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,567 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/ef0c40fd60314fba88a61e3f4debfbd8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/af01e07fa0d245f6818b261982841c14, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a1acaa34a5b84646933b065f4df477c8] 2024-11-11T16:15:00,568 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5bd3f87caf4cacbe91bad5ddbc962e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731341696640 2024-11-11T16:15:00,568 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ab6e899462c435face6bed486505fc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731341698827 2024-11-11T16:15:00,569 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef0c40fd60314fba88a61e3f4debfbd8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341696588 2024-11-11T16:15:00,569 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting af01e07fa0d245f6818b261982841c14, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731341696640 2024-11-11T16:15:00,570 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1acaa34a5b84646933b065f4df477c8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731341698827 2024-11-11T16:15:00,579 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:00,580 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/79ed526be5264e019712423432a800b8 is 50, key is test_row_0/B:col10/1731341698827/Put/seqid=0 2024-11-11T16:15:00,596 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:00,617 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111122d4832f4f9246e48ecdbd92cdb30b87_c44034eee8380292fc203bc6a539d00c store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:00,622 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111122d4832f4f9246e48ecdbd92cdb30b87_c44034eee8380292fc203bc6a539d00c, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:00,622 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111122d4832f4f9246e48ecdbd92cdb30b87_c44034eee8380292fc203bc6a539d00c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:00,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742002_1178 (size=12561) 2024-11-11T16:15:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:00,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:15:00,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,658 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/79ed526be5264e019712423432a800b8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/79ed526be5264e019712423432a800b8 2024-11-11T16:15:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,666 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/B of c44034eee8380292fc203bc6a539d00c into 79ed526be5264e019712423432a800b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:00,667 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:00,667 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/B, priority=13, startTime=1731341700551; duration=0sec 2024-11-11T16:15:00,667 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:00,667 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:B 2024-11-11T16:15:00,667 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:00,668 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:00,668 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/C is initiating minor compaction (all files) 2024-11-11T16:15:00,668 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/C in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,668 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0b113f82d3514d8a8a01463fe2d170bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/730a5e21a2724dc28b267ebd2d2a3e3c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/69c8faa69e11441f8ac2776ceaaaa158] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=35.9 K 2024-11-11T16:15:00,669 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b113f82d3514d8a8a01463fe2d170bb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341696588 2024-11-11T16:15:00,670 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 730a5e21a2724dc28b267ebd2d2a3e3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731341696640 2024-11-11T16:15:00,671 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c8faa69e11441f8ac2776ceaaaa158, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731341698827 2024-11-11T16:15:00,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742003_1179 (size=4469) 2024-11-11T16:15:00,696 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#A#compaction#151 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:00,697 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/2eef71a387ed414dab741c5d5c31f8dc is 175, key is test_row_0/A:col10/1731341698827/Put/seqid=0 2024-11-11T16:15:00,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:00,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:00,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:00,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:00,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:00,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:00,707 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:00,708 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/67f3abf54e95478e88713dd9045d17d2 is 50, key is test_row_0/C:col10/1731341698827/Put/seqid=0 2024-11-11T16:15:00,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742004_1180 (size=31515) 2024-11-11T16:15:00,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e2e26ac4cbf54c928ee94a98cda87eac_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341700653/Put/seqid=0 2024-11-11T16:15:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742005_1181 (size=12561) 2024-11-11T16:15:00,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742006_1182 (size=12304) 2024-11-11T16:15:00,745 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:00,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e2e26ac4cbf54c928ee94a98cda87eac_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e2e26ac4cbf54c928ee94a98cda87eac_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:00,753 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/67f3abf54e95478e88713dd9045d17d2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/67f3abf54e95478e88713dd9045d17d2 2024-11-11T16:15:00,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/183dc34c228f421ab60ee7a6569c1b03, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:00,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/183dc34c228f421ab60ee7a6569c1b03 is 175, key is test_row_0/A:col10/1731341700653/Put/seqid=0 2024-11-11T16:15:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341760755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341760755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341760757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341760758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,774 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/C of c44034eee8380292fc203bc6a539d00c into 67f3abf54e95478e88713dd9045d17d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:00,774 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:00,774 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/C, priority=13, startTime=1731341700551; duration=0sec 2024-11-11T16:15:00,775 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:00,775 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:C 2024-11-11T16:15:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341760766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742007_1183 (size=31105) 2024-11-11T16:15:00,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/183dc34c228f421ab60ee7a6569c1b03 2024-11-11T16:15:00,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/5ee6db9a6e8940909e910c6aea1714f6 is 50, key is test_row_0/B:col10/1731341700653/Put/seqid=0 2024-11-11T16:15:00,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742008_1184 (size=12151) 2024-11-11T16:15:00,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/5ee6db9a6e8940909e910c6aea1714f6 2024-11-11T16:15:00,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/d9fe8834ed7d4a56aab56ae366132507 is 50, key is test_row_0/C:col10/1731341700653/Put/seqid=0 2024-11-11T16:15:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341760876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341760876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341760876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341760876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742009_1185 (size=12151) 2024-11-11T16:15:00,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341760886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:00,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:00,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:00,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:00,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341761087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341761087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341761087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341761087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341761101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,150 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/2eef71a387ed414dab741c5d5c31f8dc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2eef71a387ed414dab741c5d5c31f8dc 2024-11-11T16:15:01,166 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/A of c44034eee8380292fc203bc6a539d00c into 2eef71a387ed414dab741c5d5c31f8dc(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:01,166 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:01,166 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/A, priority=13, startTime=1731341700550; duration=0sec 2024-11-11T16:15:01,166 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:01,166 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:A 2024-11-11T16:15:01,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/d9fe8834ed7d4a56aab56ae366132507 2024-11-11T16:15:01,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/183dc34c228f421ab60ee7a6569c1b03 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/183dc34c228f421ab60ee7a6569c1b03 2024-11-11T16:15:01,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/183dc34c228f421ab60ee7a6569c1b03, entries=150, sequenceid=192, filesize=30.4 K 2024-11-11T16:15:01,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/5ee6db9a6e8940909e910c6aea1714f6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/5ee6db9a6e8940909e910c6aea1714f6 2024-11-11T16:15:01,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/5ee6db9a6e8940909e910c6aea1714f6, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T16:15:01,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/d9fe8834ed7d4a56aab56ae366132507 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d9fe8834ed7d4a56aab56ae366132507 2024-11-11T16:15:01,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d9fe8834ed7d4a56aab56ae366132507, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T16:15:01,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c44034eee8380292fc203bc6a539d00c in 685ms, sequenceid=192, compaction requested=false 2024-11-11T16:15:01,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:01,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:01,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111156e5b08ad86e475c838f8d2d437a880f_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341701408/Put/seqid=0 2024-11-11T16:15:01,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742010_1186 (size=17284) 2024-11-11T16:15:01,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:01,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341761438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341761440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341761442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341761444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341761446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341761555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341761555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341761556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341761556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341761556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T16:15:01,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:01,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:01,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341761766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341761775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341761776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341761776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341761776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:01,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:01,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111156e5b08ad86e475c838f8d2d437a880f_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156e5b08ad86e475c838f8d2d437a880f_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:01,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/80afd025195748ffb2822f13a1f7197b, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:01,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/80afd025195748ffb2822f13a1f7197b is 175, key is test_row_0/A:col10/1731341701408/Put/seqid=0 2024-11-11T16:15:01,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742011_1187 (size=48389) 2024-11-11T16:15:01,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/80afd025195748ffb2822f13a1f7197b 2024-11-11T16:15:01,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/23a5f5825c3f4b9cbc5170f771d78a87 is 50, key is test_row_0/B:col10/1731341701408/Put/seqid=0 2024-11-11T16:15:01,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742012_1188 (size=12151) 2024-11-11T16:15:01,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/23a5f5825c3f4b9cbc5170f771d78a87 2024-11-11T16:15:01,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/8126355a4f6446d091603ba3d227e7dc is 50, key is test_row_0/C:col10/1731341701408/Put/seqid=0 2024-11-11T16:15:01,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:01,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:01,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742013_1189 (size=12151) 2024-11-11T16:15:01,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/8126355a4f6446d091603ba3d227e7dc 2024-11-11T16:15:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/80afd025195748ffb2822f13a1f7197b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/80afd025195748ffb2822f13a1f7197b 2024-11-11T16:15:01,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/80afd025195748ffb2822f13a1f7197b, entries=250, sequenceid=209, filesize=47.3 K 2024-11-11T16:15:01,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/23a5f5825c3f4b9cbc5170f771d78a87 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/23a5f5825c3f4b9cbc5170f771d78a87 2024-11-11T16:15:01,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/23a5f5825c3f4b9cbc5170f771d78a87, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T16:15:01,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/8126355a4f6446d091603ba3d227e7dc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/8126355a4f6446d091603ba3d227e7dc 2024-11-11T16:15:01,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/8126355a4f6446d091603ba3d227e7dc, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T16:15:01,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c44034eee8380292fc203bc6a539d00c in 560ms, sequenceid=209, compaction requested=true 2024-11-11T16:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:01,968 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:01,968 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:01,970 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:01,970 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:01,970 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/A is initiating minor compaction (all files) 2024-11-11T16:15:01,970 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/B is initiating minor compaction (all files) 2024-11-11T16:15:01,970 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/A in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,970 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/B in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,970 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2eef71a387ed414dab741c5d5c31f8dc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/183dc34c228f421ab60ee7a6569c1b03, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/80afd025195748ffb2822f13a1f7197b] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=108.4 K 2024-11-11T16:15:01,970 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/79ed526be5264e019712423432a800b8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/5ee6db9a6e8940909e910c6aea1714f6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/23a5f5825c3f4b9cbc5170f771d78a87] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=36.0 K 2024-11-11T16:15:01,970 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:01,970 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2eef71a387ed414dab741c5d5c31f8dc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/183dc34c228f421ab60ee7a6569c1b03, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/80afd025195748ffb2822f13a1f7197b] 2024-11-11T16:15:01,971 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 79ed526be5264e019712423432a800b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731341698827 2024-11-11T16:15:01,971 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eef71a387ed414dab741c5d5c31f8dc, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731341698827 2024-11-11T16:15:01,971 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee6db9a6e8940909e910c6aea1714f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731341699500 2024-11-11T16:15:01,971 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 183dc34c228f421ab60ee7a6569c1b03, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731341699500 2024-11-11T16:15:01,971 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a5f5825c3f4b9cbc5170f771d78a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731341700699 2024-11-11T16:15:01,972 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80afd025195748ffb2822f13a1f7197b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731341700699 2024-11-11T16:15:01,984 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:01,984 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/e16e6fe0e5714745aa976a613dd086e9 is 50, key is test_row_0/B:col10/1731341701408/Put/seqid=0 2024-11-11T16:15:01,994 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:02,022 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111119574ff88e364214a6d76282070b89d3_c44034eee8380292fc203bc6a539d00c store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:02,024 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111119574ff88e364214a6d76282070b89d3_c44034eee8380292fc203bc6a539d00c, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:02,024 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111119574ff88e364214a6d76282070b89d3_c44034eee8380292fc203bc6a539d00c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:02,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742014_1190 (size=12663) 2024-11-11T16:15:02,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T16:15:02,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:02,054 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T16:15:02,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:02,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:02,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:02,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:02,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:02,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:02,057 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/e16e6fe0e5714745aa976a613dd086e9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e16e6fe0e5714745aa976a613dd086e9 2024-11-11T16:15:02,069 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/B of c44034eee8380292fc203bc6a539d00c into e16e6fe0e5714745aa976a613dd086e9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:02,069 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:02,069 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/B, priority=13, startTime=1731341701968; duration=0sec 2024-11-11T16:15:02,070 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:02,070 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:B 2024-11-11T16:15:02,070 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:02,071 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:02,071 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/C is initiating minor compaction (all files) 2024-11-11T16:15:02,071 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/C in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:02,072 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/67f3abf54e95478e88713dd9045d17d2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d9fe8834ed7d4a56aab56ae366132507, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/8126355a4f6446d091603ba3d227e7dc] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=36.0 K 2024-11-11T16:15:02,072 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f3abf54e95478e88713dd9045d17d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731341698827 2024-11-11T16:15:02,073 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d9fe8834ed7d4a56aab56ae366132507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731341699500 2024-11-11T16:15:02,073 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8126355a4f6446d091603ba3d227e7dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731341700699 2024-11-11T16:15:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742015_1191 (size=4469) 2024-11-11T16:15:02,083 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#A#compaction#160 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:02,083 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/2fdc54bf2e6444f7a2814b2372f40343 is 175, key is test_row_0/A:col10/1731341701408/Put/seqid=0 2024-11-11T16:15:02,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114ba517f47842434c8010b2f8533b4e22_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341701441/Put/seqid=0 2024-11-11T16:15:02,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:02,102 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:02,103 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a2715b99ec1643eb95d9f857fdaf20fb is 50, key is test_row_0/C:col10/1731341701408/Put/seqid=0 2024-11-11T16:15:02,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341762106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341762108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341762111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341762115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341762117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742018_1194 (size=12663) 2024-11-11T16:15:02,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742017_1193 (size=12304) 2024-11-11T16:15:02,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742016_1192 (size=31617) 2024-11-11T16:15:02,198 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a2715b99ec1643eb95d9f857fdaf20fb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a2715b99ec1643eb95d9f857fdaf20fb 2024-11-11T16:15:02,207 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/C of c44034eee8380292fc203bc6a539d00c into a2715b99ec1643eb95d9f857fdaf20fb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:02,207 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:02,207 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/C, priority=13, startTime=1731341701968; duration=0sec 2024-11-11T16:15:02,207 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:02,207 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:C 2024-11-11T16:15:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341762226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341762239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341762239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341762240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341762241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341762436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341762447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341762456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341762456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341762456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:02,590 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114ba517f47842434c8010b2f8533b4e22_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114ba517f47842434c8010b2f8533b4e22_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:02,596 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/2fdc54bf2e6444f7a2814b2372f40343 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2fdc54bf2e6444f7a2814b2372f40343 2024-11-11T16:15:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5587c3e7cf7340b492dc719147a2e1bb, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:02,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5587c3e7cf7340b492dc719147a2e1bb is 175, key is test_row_0/A:col10/1731341701441/Put/seqid=0 2024-11-11T16:15:02,605 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c44034eee8380292fc203bc6a539d00c/A of c44034eee8380292fc203bc6a539d00c into 2fdc54bf2e6444f7a2814b2372f40343(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:02,605 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:02,605 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/A, priority=13, startTime=1731341701968; duration=0sec 2024-11-11T16:15:02,605 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:02,605 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:A 2024-11-11T16:15:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742019_1195 (size=31105) 2024-11-11T16:15:02,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341762746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341762756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341762766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341762766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341762766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:03,046 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5587c3e7cf7340b492dc719147a2e1bb 2024-11-11T16:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/fe4c19cc0ef6493d9e0119b94ffe0e23 is 50, key is test_row_0/B:col10/1731341701441/Put/seqid=0 2024-11-11T16:15:03,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742020_1196 (size=12151) 2024-11-11T16:15:03,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341763256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341763276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341763277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:03,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341763285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341763285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:03,473 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/fe4c19cc0ef6493d9e0119b94ffe0e23 2024-11-11T16:15:03,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/f6c475efc41343cdbc190552e895d0a0 is 50, key is test_row_0/C:col10/1731341701441/Put/seqid=0 2024-11-11T16:15:03,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742021_1197 (size=12151) 2024-11-11T16:15:03,503 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/f6c475efc41343cdbc190552e895d0a0 2024-11-11T16:15:03,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5587c3e7cf7340b492dc719147a2e1bb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5587c3e7cf7340b492dc719147a2e1bb 2024-11-11T16:15:03,512 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5587c3e7cf7340b492dc719147a2e1bb, entries=150, sequenceid=227, filesize=30.4 K 2024-11-11T16:15:03,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/fe4c19cc0ef6493d9e0119b94ffe0e23 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fe4c19cc0ef6493d9e0119b94ffe0e23 2024-11-11T16:15:03,517 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fe4c19cc0ef6493d9e0119b94ffe0e23, entries=150, sequenceid=227, filesize=11.9 K 2024-11-11T16:15:03,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/f6c475efc41343cdbc190552e895d0a0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/f6c475efc41343cdbc190552e895d0a0 2024-11-11T16:15:03,524 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/f6c475efc41343cdbc190552e895d0a0, entries=150, sequenceid=227, filesize=11.9 K 2024-11-11T16:15:03,525 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c44034eee8380292fc203bc6a539d00c in 1471ms, sequenceid=227, compaction requested=false 2024-11-11T16:15:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T16:15:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T16:15:03,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T16:15:03,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1060 sec 2024-11-11T16:15:03,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 4.1120 sec 2024-11-11T16:15:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T16:15:03,586 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:03,587 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T16:15:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T16:15:03,589 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:03,590 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:03,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T16:15:03,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T16:15:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:03,743 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T16:15:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:03,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112dfa717d542a4874b53fd9369d7c5b58_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341702110/Put/seqid=0 2024-11-11T16:15:03,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742022_1198 (size=12304) 2024-11-11T16:15:03,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:03,781 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112dfa717d542a4874b53fd9369d7c5b58_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112dfa717d542a4874b53fd9369d7c5b58_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/53cca8bc3df84b64ad18c3905edb4984, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:03,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/53cca8bc3df84b64ad18c3905edb4984 is 175, key is test_row_0/A:col10/1731341702110/Put/seqid=0 2024-11-11T16:15:03,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742023_1199 (size=31105) 2024-11-11T16:15:03,798 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/53cca8bc3df84b64ad18c3905edb4984 2024-11-11T16:15:03,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/509f152f81fd43f2856644f0f69d2b86 is 50, key is test_row_0/B:col10/1731341702110/Put/seqid=0 2024-11-11T16:15:03,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742024_1200 (size=12151) 2024-11-11T16:15:03,836 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/509f152f81fd43f2856644f0f69d2b86 2024-11-11T16:15:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/7301ae1019ff4be1a2e519b7afd1aa89 is 50, key is test_row_0/C:col10/1731341702110/Put/seqid=0 2024-11-11T16:15:03,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742025_1201 (size=12151) 2024-11-11T16:15:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T16:15:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T16:15:04,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:04,287 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/7301ae1019ff4be1a2e519b7afd1aa89 2024-11-11T16:15:04,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/53cca8bc3df84b64ad18c3905edb4984 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/53cca8bc3df84b64ad18c3905edb4984 2024-11-11T16:15:04,313 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/53cca8bc3df84b64ad18c3905edb4984, entries=150, sequenceid=244, filesize=30.4 K 2024-11-11T16:15:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/509f152f81fd43f2856644f0f69d2b86 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/509f152f81fd43f2856644f0f69d2b86 2024-11-11T16:15:04,325 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/509f152f81fd43f2856644f0f69d2b86, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T16:15:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/7301ae1019ff4be1a2e519b7afd1aa89 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/7301ae1019ff4be1a2e519b7afd1aa89 2024-11-11T16:15:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,339 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/7301ae1019ff4be1a2e519b7afd1aa89, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T16:15:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,342 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=73.80 KB/75570 for c44034eee8380292fc203bc6a539d00c in 599ms, sequenceid=244, compaction requested=true 2024-11-11T16:15:04,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:04,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:04,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T16:15:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T16:15:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T16:15:04,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 755 msec 2024-11-11T16:15:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 763 msec 2024-11-11T16:15:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:15:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112c7a909f13024653a37e1b6f2db13dee_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_1/A:col10/1731341704338/Put/seqid=0 2024-11-11T16:15:04,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341764466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341764472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341764473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341764478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341764480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742027_1203 (size=24858) 2024-11-11T16:15:04,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341764586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341764586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341764596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341764596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341764596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T16:15:04,726 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:04,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T16:15:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T16:15:04,729 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:04,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:04,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:04,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341764806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341764806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341764817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341764817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341764818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T16:15:04,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:04,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:04,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:04,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112c7a909f13024653a37e1b6f2db13dee_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112c7a909f13024653a37e1b6f2db13dee_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:04,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/66586ede26ed4cca964389e45a040981, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:04,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/66586ede26ed4cca964389e45a040981 is 175, key is test_row_1/A:col10/1731341704338/Put/seqid=0 2024-11-11T16:15:04,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742026_1202 (size=74491) 2024-11-11T16:15:04,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/66586ede26ed4cca964389e45a040981 2024-11-11T16:15:04,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/bac9bf2bddff4a518bbacbe3da8c0aa7 is 50, key is test_row_1/B:col10/1731341704338/Put/seqid=0 2024-11-11T16:15:04,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742028_1204 (size=9857) 2024-11-11T16:15:04,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/bac9bf2bddff4a518bbacbe3da8c0aa7 2024-11-11T16:15:05,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a6f530b1ad514639852378dadebac8ba is 50, key is test_row_1/C:col10/1731341704338/Put/seqid=0 2024-11-11T16:15:05,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:05,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:05,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T16:15:05,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742029_1205 (size=9857) 2024-11-11T16:15:05,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a6f530b1ad514639852378dadebac8ba 2024-11-11T16:15:05,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/66586ede26ed4cca964389e45a040981 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/66586ede26ed4cca964389e45a040981 2024-11-11T16:15:05,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/66586ede26ed4cca964389e45a040981, entries=400, sequenceid=259, filesize=72.7 K 2024-11-11T16:15:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/bac9bf2bddff4a518bbacbe3da8c0aa7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/bac9bf2bddff4a518bbacbe3da8c0aa7 2024-11-11T16:15:05,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/bac9bf2bddff4a518bbacbe3da8c0aa7, entries=100, sequenceid=259, filesize=9.6 K 2024-11-11T16:15:05,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a6f530b1ad514639852378dadebac8ba as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a6f530b1ad514639852378dadebac8ba 2024-11-11T16:15:05,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a6f530b1ad514639852378dadebac8ba, entries=100, sequenceid=259, filesize=9.6 K 2024-11-11T16:15:05,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c44034eee8380292fc203bc6a539d00c in 695ms, sequenceid=259, compaction requested=true 2024-11-11T16:15:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:05,112 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:05,114 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 168318 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:05,115 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/A is initiating minor compaction (all files) 2024-11-11T16:15:05,115 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/A in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,115 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2fdc54bf2e6444f7a2814b2372f40343, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5587c3e7cf7340b492dc719147a2e1bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/53cca8bc3df84b64ad18c3905edb4984, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/66586ede26ed4cca964389e45a040981] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=164.4 K 2024-11-11T16:15:05,115 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,115 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2fdc54bf2e6444f7a2814b2372f40343, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5587c3e7cf7340b492dc719147a2e1bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/53cca8bc3df84b64ad18c3905edb4984, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/66586ede26ed4cca964389e45a040981] 2024-11-11T16:15:05,116 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fdc54bf2e6444f7a2814b2372f40343, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731341700699 2024-11-11T16:15:05,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:05,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:05,116 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:05,117 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5587c3e7cf7340b492dc719147a2e1bb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731341701421 2024-11-11T16:15:05,117 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53cca8bc3df84b64ad18c3905edb4984, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731341702095 2024-11-11T16:15:05,118 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66586ede26ed4cca964389e45a040981, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731341704309 2024-11-11T16:15:05,119 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46822 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:05,119 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/B is initiating minor compaction (all files) 2024-11-11T16:15:05,119 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/B in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,119 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e16e6fe0e5714745aa976a613dd086e9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fe4c19cc0ef6493d9e0119b94ffe0e23, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/509f152f81fd43f2856644f0f69d2b86, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/bac9bf2bddff4a518bbacbe3da8c0aa7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=45.7 K 2024-11-11T16:15:05,120 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e16e6fe0e5714745aa976a613dd086e9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731341700699 2024-11-11T16:15:05,121 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting fe4c19cc0ef6493d9e0119b94ffe0e23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731341701421 2024-11-11T16:15:05,122 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 509f152f81fd43f2856644f0f69d2b86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731341702095 2024-11-11T16:15:05,123 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting bac9bf2bddff4a518bbacbe3da8c0aa7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731341704338 2024-11-11T16:15:05,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:05,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:15:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:05,145 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:05,156 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:05,157 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/e8d12e6f8d324933b72514ce4b0ef5a1 is 50, key is test_row_0/B:col10/1731341702110/Put/seqid=0 2024-11-11T16:15:05,167 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111174b9289a6d764918bbd2739acff3f570_c44034eee8380292fc203bc6a539d00c store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:05,177 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111174b9289a6d764918bbd2739acff3f570_c44034eee8380292fc203bc6a539d00c, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:05,177 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111174b9289a6d764918bbd2739acff3f570_c44034eee8380292fc203bc6a539d00c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:05,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111172df837c7de54c429fec5be71f296bfb_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341704473/Put/seqid=0 2024-11-11T16:15:05,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742030_1206 (size=12899) 2024-11-11T16:15:05,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,199 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/e8d12e6f8d324933b72514ce4b0ef5a1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e8d12e6f8d324933b72514ce4b0ef5a1 2024-11-11T16:15:05,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341765179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,206 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c44034eee8380292fc203bc6a539d00c/B of c44034eee8380292fc203bc6a539d00c into e8d12e6f8d324933b72514ce4b0ef5a1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:05,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:05,206 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/B, priority=12, startTime=1731341705116; duration=0sec 2024-11-11T16:15:05,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:05,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:B 2024-11-11T16:15:05,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:05,208 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46822 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:05,208 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/C is initiating minor compaction (all files) 2024-11-11T16:15:05,208 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/C in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,208 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a2715b99ec1643eb95d9f857fdaf20fb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/f6c475efc41343cdbc190552e895d0a0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/7301ae1019ff4be1a2e519b7afd1aa89, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a6f530b1ad514639852378dadebac8ba] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=45.7 K 2024-11-11T16:15:05,208 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a2715b99ec1643eb95d9f857fdaf20fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731341700699 2024-11-11T16:15:05,209 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f6c475efc41343cdbc190552e895d0a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731341701421 2024-11-11T16:15:05,209 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7301ae1019ff4be1a2e519b7afd1aa89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731341702095 2024-11-11T16:15:05,210 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a6f530b1ad514639852378dadebac8ba, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731341704338 2024-11-11T16:15:05,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341765198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341765201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341765203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341765204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742031_1207 (size=4469) 2024-11-11T16:15:05,228 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#A#compaction#171 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:05,229 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/2cd4803660e540a7900864cf9e370a5b is 175, key is test_row_0/A:col10/1731341702110/Put/seqid=0 2024-11-11T16:15:05,244 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:05,244 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/9db376d453ae495aa0a8b4f1e832a8bb is 50, key is test_row_0/C:col10/1731341702110/Put/seqid=0 2024-11-11T16:15:05,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742032_1208 (size=12454) 2024-11-11T16:15:05,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742034_1210 (size=12899) 2024-11-11T16:15:05,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742033_1209 (size=31960) 2024-11-11T16:15:05,316 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/9db376d453ae495aa0a8b4f1e832a8bb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/9db376d453ae495aa0a8b4f1e832a8bb 2024-11-11T16:15:05,325 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/2cd4803660e540a7900864cf9e370a5b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2cd4803660e540a7900864cf9e370a5b 2024-11-11T16:15:05,328 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c44034eee8380292fc203bc6a539d00c/C of c44034eee8380292fc203bc6a539d00c into 9db376d453ae495aa0a8b4f1e832a8bb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:05,328 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:05,328 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/C, priority=12, startTime=1731341705134; duration=0sec 2024-11-11T16:15:05,329 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:05,329 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:C 2024-11-11T16:15:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341765316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,341 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c44034eee8380292fc203bc6a539d00c/A of c44034eee8380292fc203bc6a539d00c into 2cd4803660e540a7900864cf9e370a5b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:05,341 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:05,341 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/A, priority=12, startTime=1731341705112; duration=0sec 2024-11-11T16:15:05,342 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:05,342 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:A 2024-11-11T16:15:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341765327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341765332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341765327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341765326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:05,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:05,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T16:15:05,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:05,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:05,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341765545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341765555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341765556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341765556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341765556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:05,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111172df837c7de54c429fec5be71f296bfb_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111172df837c7de54c429fec5be71f296bfb_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:05,654 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5bb89953b3634f37aa95758e96e49039, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:05,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5bb89953b3634f37aa95758e96e49039 is 175, key is test_row_0/A:col10/1731341704473/Put/seqid=0 2024-11-11T16:15:05,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742035_1211 (size=31255) 2024-11-11T16:15:05,662 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5bb89953b3634f37aa95758e96e49039 2024-11-11T16:15:05,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/a9ac08cfb582426baf40e18026e540fc is 50, key is test_row_0/B:col10/1731341704473/Put/seqid=0 2024-11-11T16:15:05,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742036_1212 (size=12301) 2024-11-11T16:15:05,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/a9ac08cfb582426baf40e18026e540fc 2024-11-11T16:15:05,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/bf599385ec9c43b282b880977ab43d03 is 50, key is test_row_0/C:col10/1731341704473/Put/seqid=0 2024-11-11T16:15:05,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742037_1213 (size=12301) 2024-11-11T16:15:05,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341765856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T16:15:05,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341765865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341765865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341765866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341765866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:05,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:05,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:05,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:05,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:06,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/bf599385ec9c43b282b880977ab43d03 2024-11-11T16:15:06,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5bb89953b3634f37aa95758e96e49039 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5bb89953b3634f37aa95758e96e49039 2024-11-11T16:15:06,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:06,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:06,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:06,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5bb89953b3634f37aa95758e96e49039, entries=150, sequenceid=278, filesize=30.5 K 2024-11-11T16:15:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/a9ac08cfb582426baf40e18026e540fc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/a9ac08cfb582426baf40e18026e540fc 2024-11-11T16:15:06,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/a9ac08cfb582426baf40e18026e540fc, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T16:15:06,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/bf599385ec9c43b282b880977ab43d03 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/bf599385ec9c43b282b880977ab43d03 2024-11-11T16:15:06,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/bf599385ec9c43b282b880977ab43d03, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T16:15:06,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c44034eee8380292fc203bc6a539d00c in 1007ms, sequenceid=278, compaction requested=false 2024-11-11T16:15:06,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:06,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T16:15:06,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:06,282 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:15:06,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:06,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:06,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:06,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:06,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ebe5805a65e74fd49a00d81ce5856e11_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341705201/Put/seqid=0 2024-11-11T16:15:06,364 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T16:15:06,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. as already flushing 2024-11-11T16:15:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:06,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742038_1214 (size=12454) 2024-11-11T16:15:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:06,404 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ebe5805a65e74fd49a00d81ce5856e11_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebe5805a65e74fd49a00d81ce5856e11_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:06,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/c1f2d3961cc549949110b6c592468556, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/c1f2d3961cc549949110b6c592468556 is 175, key is test_row_0/A:col10/1731341705201/Put/seqid=0 2024-11-11T16:15:06,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341766409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341766409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341766409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341766411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341766414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742039_1215 (size=31255) 2024-11-11T16:15:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341766527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341766527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341766528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341766530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341766530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341766747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341766748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341766748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341766748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341766755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:06,859 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/c1f2d3961cc549949110b6c592468556 2024-11-11T16:15:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T16:15:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/2c62593f94964983bc81094b9761e7b6 is 50, key is test_row_0/B:col10/1731341705201/Put/seqid=0 2024-11-11T16:15:06,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742040_1216 (size=12301) 2024-11-11T16:15:06,972 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/2c62593f94964983bc81094b9761e7b6 2024-11-11T16:15:06,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/43eac536e30d43aa98ee840162e2add5 is 50, key is test_row_0/C:col10/1731341705201/Put/seqid=0 2024-11-11T16:15:07,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742041_1217 (size=12301) 2024-11-11T16:15:07,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341767066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341767068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341767073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341767074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341767073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,447 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/43eac536e30d43aa98ee840162e2add5 2024-11-11T16:15:07,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/c1f2d3961cc549949110b6c592468556 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c1f2d3961cc549949110b6c592468556 2024-11-11T16:15:07,461 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c1f2d3961cc549949110b6c592468556, entries=150, sequenceid=294, filesize=30.5 K 2024-11-11T16:15:07,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/2c62593f94964983bc81094b9761e7b6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/2c62593f94964983bc81094b9761e7b6 2024-11-11T16:15:07,468 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/2c62593f94964983bc81094b9761e7b6, entries=150, sequenceid=294, filesize=12.0 K 2024-11-11T16:15:07,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/43eac536e30d43aa98ee840162e2add5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/43eac536e30d43aa98ee840162e2add5 2024-11-11T16:15:07,482 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/43eac536e30d43aa98ee840162e2add5, entries=150, sequenceid=294, filesize=12.0 K 2024-11-11T16:15:07,486 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c44034eee8380292fc203bc6a539d00c in 1204ms, sequenceid=294, compaction requested=true 2024-11-11T16:15:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T16:15:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T16:15:07,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T16:15:07,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7580 sec 2024-11-11T16:15:07,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.7640 sec 2024-11-11T16:15:07,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:15:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:07,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f3847769d32346fd9c112ee89f8ba7c1_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341706400/Put/seqid=0 2024-11-11T16:15:07,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341767615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341767618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341767621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341767630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341767635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742042_1218 (size=14994) 2024-11-11T16:15:07,680 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:07,686 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f3847769d32346fd9c112ee89f8ba7c1_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f3847769d32346fd9c112ee89f8ba7c1_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:07,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5c9efb325b1146c8b5d61a960f12b12f, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:07,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5c9efb325b1146c8b5d61a960f12b12f is 175, key is test_row_0/A:col10/1731341706400/Put/seqid=0 2024-11-11T16:15:07,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742043_1219 (size=39949) 2024-11-11T16:15:07,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341767736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341767745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341767746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341767756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341767756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,817 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-11T16:15:07,817 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-11T16:15:07,817 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:07,817 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,817 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:07,817 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:07,818 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:07,818 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,822 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-11T16:15:07,822 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:07,822 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,822 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:07,827 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-11T16:15:07,827 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:07,827 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,827 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:07,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:07,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341767945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341767956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341767956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341767966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341767966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5c9efb325b1146c8b5d61a960f12b12f 2024-11-11T16:15:08,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/ce5147180f9f4107a0efe04df658f6e3 is 50, key is test_row_0/B:col10/1731341706400/Put/seqid=0 2024-11-11T16:15:08,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742044_1220 (size=12301) 2024-11-11T16:15:08,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341768256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341768265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341768265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341768276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341768276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/ce5147180f9f4107a0efe04df658f6e3 2024-11-11T16:15:08,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/86a06d5d053f463cb359753458f5f3ae is 50, key is test_row_0/C:col10/1731341706400/Put/seqid=0 2024-11-11T16:15:08,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742045_1221 (size=12301) 2024-11-11T16:15:08,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58222 deadline: 1731341768765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58246 deadline: 1731341768776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58228 deadline: 1731341768776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1731341768786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58214 deadline: 1731341768786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T16:15:08,896 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:08,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/86a06d5d053f463cb359753458f5f3ae 2024-11-11T16:15:08,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/5c9efb325b1146c8b5d61a960f12b12f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5c9efb325b1146c8b5d61a960f12b12f 2024-11-11T16:15:08,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5c9efb325b1146c8b5d61a960f12b12f, entries=200, sequenceid=314, filesize=39.0 K 2024-11-11T16:15:08,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/ce5147180f9f4107a0efe04df658f6e3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ce5147180f9f4107a0efe04df658f6e3 2024-11-11T16:15:08,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ce5147180f9f4107a0efe04df658f6e3, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T16:15:08,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/86a06d5d053f463cb359753458f5f3ae as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/86a06d5d053f463cb359753458f5f3ae 2024-11-11T16:15:09,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/86a06d5d053f463cb359753458f5f3ae, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T16:15:09,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c44034eee8380292fc203bc6a539d00c in 1414ms, sequenceid=314, compaction requested=true 2024-11-11T16:15:09,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:09,004 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:09,005 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:09,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c44034eee8380292fc203bc6a539d00c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:09,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:09,006 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134419 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:09,006 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/A is initiating minor compaction (all files) 2024-11-11T16:15:09,006 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/A in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:09,007 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2cd4803660e540a7900864cf9e370a5b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5bb89953b3634f37aa95758e96e49039, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c1f2d3961cc549949110b6c592468556, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5c9efb325b1146c8b5d61a960f12b12f] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=131.3 K 2024-11-11T16:15:09,007 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:09,007 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2cd4803660e540a7900864cf9e370a5b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5bb89953b3634f37aa95758e96e49039, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c1f2d3961cc549949110b6c592468556, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5c9efb325b1146c8b5d61a960f12b12f] 2024-11-11T16:15:09,009 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd4803660e540a7900864cf9e370a5b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731341702110 2024-11-11T16:15:09,009 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb89953b3634f37aa95758e96e49039, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731341704473 2024-11-11T16:15:09,010 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:09,010 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/B is initiating minor compaction (all files) 2024-11-11T16:15:09,010 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/B in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:09,010 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e8d12e6f8d324933b72514ce4b0ef5a1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/a9ac08cfb582426baf40e18026e540fc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/2c62593f94964983bc81094b9761e7b6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ce5147180f9f4107a0efe04df658f6e3] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=48.6 K 2024-11-11T16:15:09,010 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d12e6f8d324933b72514ce4b0ef5a1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731341702110 2024-11-11T16:15:09,010 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f2d3961cc549949110b6c592468556, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731341705189 2024-11-11T16:15:09,011 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ac08cfb582426baf40e18026e540fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731341704473 2024-11-11T16:15:09,011 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c9efb325b1146c8b5d61a960f12b12f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731341706393 2024-11-11T16:15:09,015 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c62593f94964983bc81094b9761e7b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731341705189 2024-11-11T16:15:09,015 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5147180f9f4107a0efe04df658f6e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731341706393 2024-11-11T16:15:09,037 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:09,042 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118aecb00aef2c4c838333068e4923089c_c44034eee8380292fc203bc6a539d00c store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:09,042 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#B#compaction#183 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:09,043 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/1353c174abb54c5296a0cc8a3661f279 is 50, key is test_row_0/B:col10/1731341706400/Put/seqid=0 2024-11-11T16:15:09,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742046_1222 (size=13085) 2024-11-11T16:15:09,086 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118aecb00aef2c4c838333068e4923089c_c44034eee8380292fc203bc6a539d00c, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:09,086 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118aecb00aef2c4c838333068e4923089c_c44034eee8380292fc203bc6a539d00c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:09,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742047_1223 (size=4469) 2024-11-11T16:15:09,466 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/1353c174abb54c5296a0cc8a3661f279 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/1353c174abb54c5296a0cc8a3661f279 2024-11-11T16:15:09,472 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c44034eee8380292fc203bc6a539d00c/B of c44034eee8380292fc203bc6a539d00c into 1353c174abb54c5296a0cc8a3661f279(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:09,472 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:09,472 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/B, priority=12, startTime=1731341709005; duration=0sec 2024-11-11T16:15:09,472 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:09,472 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:B 2024-11-11T16:15:09,473 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:09,474 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:09,474 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): c44034eee8380292fc203bc6a539d00c/C is initiating minor compaction (all files) 2024-11-11T16:15:09,474 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c44034eee8380292fc203bc6a539d00c/C in TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:09,474 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/9db376d453ae495aa0a8b4f1e832a8bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/bf599385ec9c43b282b880977ab43d03, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/43eac536e30d43aa98ee840162e2add5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/86a06d5d053f463cb359753458f5f3ae] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp, totalSize=48.6 K 2024-11-11T16:15:09,475 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db376d453ae495aa0a8b4f1e832a8bb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731341702110 2024-11-11T16:15:09,475 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf599385ec9c43b282b880977ab43d03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731341704473 2024-11-11T16:15:09,475 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43eac536e30d43aa98ee840162e2add5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731341705189 2024-11-11T16:15:09,476 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a06d5d053f463cb359753458f5f3ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731341706393 2024-11-11T16:15:09,491 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#C#compaction#185 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:09,491 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/73cf28f0908b4d09978f6c2fb69036b7 is 50, key is test_row_0/C:col10/1731341706400/Put/seqid=0 2024-11-11T16:15:09,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742048_1224 (size=13085) 2024-11-11T16:15:09,510 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c44034eee8380292fc203bc6a539d00c#A#compaction#184 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:09,511 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/6f5c9e46318b462bb0a11564813852dd is 175, key is test_row_0/A:col10/1731341706400/Put/seqid=0 2024-11-11T16:15:09,515 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/73cf28f0908b4d09978f6c2fb69036b7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/73cf28f0908b4d09978f6c2fb69036b7 2024-11-11T16:15:09,522 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c44034eee8380292fc203bc6a539d00c/C of c44034eee8380292fc203bc6a539d00c into 73cf28f0908b4d09978f6c2fb69036b7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:09,522 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:09,523 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/C, priority=12, startTime=1731341709006; duration=0sec 2024-11-11T16:15:09,523 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:09,523 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:C 2024-11-11T16:15:09,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742049_1225 (size=32039) 2024-11-11T16:15:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:09,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T16:15:09,789 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-11T16:15:09,789 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-11T16:15:09,789 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:09,789 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:09,789 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,789 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:09,789 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,789 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:09,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:09,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:09,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:09,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:09,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:09,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:09,790 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-11T16:15:09,790 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:09,790 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,790 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:09,799 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-11T16:15:09,799 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:09,799 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,800 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:09,802 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-11T16:15:09,802 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:09,802 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,802 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2171 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2288 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T16:15:09,802 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1004 2024-11-11T16:15:09,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3006 rows 2024-11-11T16:15:09,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1000 2024-11-11T16:15:09,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3000 rows 2024-11-11T16:15:09,803 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:15:09,803 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:15:09,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:09,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:09,805 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T16:15:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T16:15:09,812 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341709811"}]},"ts":"1731341709811"} 2024-11-11T16:15:09,817 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T16:15:09,817 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T16:15:09,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:15:09,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, UNASSIGN}] 2024-11-11T16:15:09,822 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, UNASSIGN 2024-11-11T16:15:09,823 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:09,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, UNASSIGN because future has completed 2024-11-11T16:15:09,825 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:15:09,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:15:09,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113c46d956ef5a4e93aca9d33edcc67183_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341709787/Put/seqid=0 2024-11-11T16:15:09,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742050_1226 (size=12454) 2024-11-11T16:15:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T16:15:09,961 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/6f5c9e46318b462bb0a11564813852dd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6f5c9e46318b462bb0a11564813852dd 2024-11-11T16:15:09,980 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c44034eee8380292fc203bc6a539d00c/A of c44034eee8380292fc203bc6a539d00c into 6f5c9e46318b462bb0a11564813852dd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:09,980 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:09,980 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., storeName=c44034eee8380292fc203bc6a539d00c/A, priority=12, startTime=1731341709004; duration=0sec 2024-11-11T16:15:09,980 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:09,980 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c44034eee8380292fc203bc6a539d00c:A 2024-11-11T16:15:09,980 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:09,981 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:15:09,981 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing c44034eee8380292fc203bc6a539d00c, disabling compactions & flushes 2024-11-11T16:15:09,981 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T16:15:10,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:10,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113c46d956ef5a4e93aca9d33edcc67183_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113c46d956ef5a4e93aca9d33edcc67183_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:10,291 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/14fc1f1d4ff94aee95c86721386e5c96, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:10,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/14fc1f1d4ff94aee95c86721386e5c96 is 175, key is test_row_0/A:col10/1731341709787/Put/seqid=0 2024-11-11T16:15:10,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742051_1227 (size=31255) 2024-11-11T16:15:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T16:15:10,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/14fc1f1d4ff94aee95c86721386e5c96 2024-11-11T16:15:10,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/6f3867ba07ff4d00b231fe055a865412 is 50, key is test_row_0/B:col10/1731341709787/Put/seqid=0 2024-11-11T16:15:10,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742052_1228 (size=12301) 2024-11-11T16:15:10,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/6f3867ba07ff4d00b231fe055a865412 2024-11-11T16:15:10,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a065aec424c34cd59a6b2ec84eff436b is 50, key is test_row_0/C:col10/1731341709787/Put/seqid=0 2024-11-11T16:15:10,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742053_1229 (size=12301) 2024-11-11T16:15:10,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a065aec424c34cd59a6b2ec84eff436b 2024-11-11T16:15:10,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/14fc1f1d4ff94aee95c86721386e5c96 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/14fc1f1d4ff94aee95c86721386e5c96 2024-11-11T16:15:10,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/14fc1f1d4ff94aee95c86721386e5c96, entries=150, sequenceid=332, filesize=30.5 K 2024-11-11T16:15:10,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/6f3867ba07ff4d00b231fe055a865412 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/6f3867ba07ff4d00b231fe055a865412 2024-11-11T16:15:10,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/6f3867ba07ff4d00b231fe055a865412, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T16:15:10,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/a065aec424c34cd59a6b2ec84eff436b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a065aec424c34cd59a6b2ec84eff436b 2024-11-11T16:15:10,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a065aec424c34cd59a6b2ec84eff436b, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T16:15:10,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for c44034eee8380292fc203bc6a539d00c in 1062ms, sequenceid=332, compaction requested=false 2024-11-11T16:15:10,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c44034eee8380292fc203bc6a539d00c: 2024-11-11T16:15:10,851 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:10,851 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:10,851 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. after waiting 0 ms 2024-11-11T16:15:10,851 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:10,851 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing c44034eee8380292fc203bc6a539d00c 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-11T16:15:10,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=A 2024-11-11T16:15:10,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:10,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=B 2024-11-11T16:15:10,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:10,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c44034eee8380292fc203bc6a539d00c, store=C 2024-11-11T16:15:10,852 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:10,866 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111063bf4ab9f814d8d9e5b8dc4c69e7eac_c44034eee8380292fc203bc6a539d00c is 50, key is test_row_0/A:col10/1731341709798/Put/seqid=0 2024-11-11T16:15:10,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742054_1230 (size=7374) 2024-11-11T16:15:10,890 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:10,896 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111063bf4ab9f814d8d9e5b8dc4c69e7eac_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111063bf4ab9f814d8d9e5b8dc4c69e7eac_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:10,897 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/cf2e5608efca4d21bdfdb23b3292240e, store: [table=TestAcidGuarantees family=A region=c44034eee8380292fc203bc6a539d00c] 2024-11-11T16:15:10,898 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/cf2e5608efca4d21bdfdb23b3292240e is 175, key is test_row_0/A:col10/1731341709798/Put/seqid=0 2024-11-11T16:15:10,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742055_1231 (size=13865) 2024-11-11T16:15:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T16:15:11,307 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/cf2e5608efca4d21bdfdb23b3292240e 2024-11-11T16:15:11,318 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/405f8175f4774854a1d06060f738831d is 50, key is test_row_0/B:col10/1731341709798/Put/seqid=0 2024-11-11T16:15:11,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742056_1232 (size=7415) 2024-11-11T16:15:11,339 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/405f8175f4774854a1d06060f738831d 2024-11-11T16:15:11,350 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/ab17b58ed0974192aadc8d84ee2eff7e is 50, key is test_row_0/C:col10/1731341709798/Put/seqid=0 2024-11-11T16:15:11,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742057_1233 (size=7415) 2024-11-11T16:15:11,369 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/ab17b58ed0974192aadc8d84ee2eff7e 2024-11-11T16:15:11,376 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/A/cf2e5608efca4d21bdfdb23b3292240e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/cf2e5608efca4d21bdfdb23b3292240e 2024-11-11T16:15:11,382 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/cf2e5608efca4d21bdfdb23b3292240e, entries=50, sequenceid=338, filesize=13.5 K 2024-11-11T16:15:11,383 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/B/405f8175f4774854a1d06060f738831d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/405f8175f4774854a1d06060f738831d 2024-11-11T16:15:11,390 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/405f8175f4774854a1d06060f738831d, entries=50, sequenceid=338, filesize=7.2 K 2024-11-11T16:15:11,391 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/.tmp/C/ab17b58ed0974192aadc8d84ee2eff7e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ab17b58ed0974192aadc8d84ee2eff7e 2024-11-11T16:15:11,396 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ab17b58ed0974192aadc8d84ee2eff7e, entries=50, sequenceid=338, filesize=7.2 K 2024-11-11T16:15:11,399 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c44034eee8380292fc203bc6a539d00c in 548ms, sequenceid=338, compaction requested=true 2024-11-11T16:15:11,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/21c717d94543407fa994b03a4688cc83, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5648459dd3cc4b499e7beaf8d63a18d3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a6d78dba99e94ab6b10275ea2d6fc224, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6d3d1f1b9ef141e68ca0c7a392f5454b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/db966dbbd879478985f2eb941a4be113, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/0e61309a0aa64764b7d2db84e5df2338, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c3fe09b7f24749c38c45b95674bc2d6e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/22779a5068b94128a4f82a8a019eca11, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/ef0c40fd60314fba88a61e3f4debfbd8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/4d9ba34c09544fae8790ebc4cb39dfeb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/af01e07fa0d245f6818b261982841c14, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2eef71a387ed414dab741c5d5c31f8dc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a1acaa34a5b84646933b065f4df477c8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/183dc34c228f421ab60ee7a6569c1b03, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/80afd025195748ffb2822f13a1f7197b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2fdc54bf2e6444f7a2814b2372f40343, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5587c3e7cf7340b492dc719147a2e1bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/53cca8bc3df84b64ad18c3905edb4984, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/66586ede26ed4cca964389e45a040981, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2cd4803660e540a7900864cf9e370a5b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5bb89953b3634f37aa95758e96e49039, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c1f2d3961cc549949110b6c592468556, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5c9efb325b1146c8b5d61a960f12b12f] to archive 2024-11-11T16:15:11,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:15:11,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/21c717d94543407fa994b03a4688cc83 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/21c717d94543407fa994b03a4688cc83 2024-11-11T16:15:11,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5648459dd3cc4b499e7beaf8d63a18d3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5648459dd3cc4b499e7beaf8d63a18d3 2024-11-11T16:15:11,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a6d78dba99e94ab6b10275ea2d6fc224 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a6d78dba99e94ab6b10275ea2d6fc224 2024-11-11T16:15:11,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6d3d1f1b9ef141e68ca0c7a392f5454b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6d3d1f1b9ef141e68ca0c7a392f5454b 2024-11-11T16:15:11,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/db966dbbd879478985f2eb941a4be113 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/db966dbbd879478985f2eb941a4be113 2024-11-11T16:15:11,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/0e61309a0aa64764b7d2db84e5df2338 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/0e61309a0aa64764b7d2db84e5df2338 2024-11-11T16:15:11,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c3fe09b7f24749c38c45b95674bc2d6e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c3fe09b7f24749c38c45b95674bc2d6e 2024-11-11T16:15:11,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/22779a5068b94128a4f82a8a019eca11 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/22779a5068b94128a4f82a8a019eca11 2024-11-11T16:15:11,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/ef0c40fd60314fba88a61e3f4debfbd8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/ef0c40fd60314fba88a61e3f4debfbd8 2024-11-11T16:15:11,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/4d9ba34c09544fae8790ebc4cb39dfeb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/4d9ba34c09544fae8790ebc4cb39dfeb 2024-11-11T16:15:11,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/af01e07fa0d245f6818b261982841c14 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/af01e07fa0d245f6818b261982841c14 2024-11-11T16:15:11,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2eef71a387ed414dab741c5d5c31f8dc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2eef71a387ed414dab741c5d5c31f8dc 2024-11-11T16:15:11,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a1acaa34a5b84646933b065f4df477c8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/a1acaa34a5b84646933b065f4df477c8 2024-11-11T16:15:11,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/183dc34c228f421ab60ee7a6569c1b03 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/183dc34c228f421ab60ee7a6569c1b03 2024-11-11T16:15:11,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/80afd025195748ffb2822f13a1f7197b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/80afd025195748ffb2822f13a1f7197b 2024-11-11T16:15:11,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2fdc54bf2e6444f7a2814b2372f40343 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2fdc54bf2e6444f7a2814b2372f40343 2024-11-11T16:15:11,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5587c3e7cf7340b492dc719147a2e1bb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5587c3e7cf7340b492dc719147a2e1bb 2024-11-11T16:15:11,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/53cca8bc3df84b64ad18c3905edb4984 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/53cca8bc3df84b64ad18c3905edb4984 2024-11-11T16:15:11,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/66586ede26ed4cca964389e45a040981 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/66586ede26ed4cca964389e45a040981 2024-11-11T16:15:11,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2cd4803660e540a7900864cf9e370a5b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/2cd4803660e540a7900864cf9e370a5b 2024-11-11T16:15:11,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5bb89953b3634f37aa95758e96e49039 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5bb89953b3634f37aa95758e96e49039 2024-11-11T16:15:11,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c1f2d3961cc549949110b6c592468556 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/c1f2d3961cc549949110b6c592468556 2024-11-11T16:15:11,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5c9efb325b1146c8b5d61a960f12b12f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/5c9efb325b1146c8b5d61a960f12b12f 2024-11-11T16:15:11,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/16abc07bf9e54275860e0371d78f73b6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e92ef4d8b6c04e77b3cdfe3a0ec02313, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fd5f993973fa4748a693cca2d608228a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/4f339a55d9b0455abd85d337524952dd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/eb8a245e2ae74f90bd28309a2f5f0508, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/c8821e6f600c48989cb184c8e7756f21, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/93287a51ffdf4bbebf814bed4fb37a52, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ad6c1d31c60e484b9cd5c98808b34516, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9735ef6fa40344af800a16153ab87f13, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/246d9935e50145149fe828698a8dcb13, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/0e5bd3f87caf4cacbe91bad5ddbc962e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/79ed526be5264e019712423432a800b8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9ab6e899462c435face6bed486505fc7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/5ee6db9a6e8940909e910c6aea1714f6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e16e6fe0e5714745aa976a613dd086e9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/23a5f5825c3f4b9cbc5170f771d78a87, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fe4c19cc0ef6493d9e0119b94ffe0e23, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/509f152f81fd43f2856644f0f69d2b86, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e8d12e6f8d324933b72514ce4b0ef5a1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/bac9bf2bddff4a518bbacbe3da8c0aa7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/a9ac08cfb582426baf40e18026e540fc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/2c62593f94964983bc81094b9761e7b6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ce5147180f9f4107a0efe04df658f6e3] to archive 2024-11-11T16:15:11,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:15:11,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/16abc07bf9e54275860e0371d78f73b6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/16abc07bf9e54275860e0371d78f73b6 2024-11-11T16:15:11,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e92ef4d8b6c04e77b3cdfe3a0ec02313 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e92ef4d8b6c04e77b3cdfe3a0ec02313 2024-11-11T16:15:11,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fd5f993973fa4748a693cca2d608228a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fd5f993973fa4748a693cca2d608228a 2024-11-11T16:15:11,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/4f339a55d9b0455abd85d337524952dd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/4f339a55d9b0455abd85d337524952dd 2024-11-11T16:15:11,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/eb8a245e2ae74f90bd28309a2f5f0508 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/eb8a245e2ae74f90bd28309a2f5f0508 2024-11-11T16:15:11,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/c8821e6f600c48989cb184c8e7756f21 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/c8821e6f600c48989cb184c8e7756f21 2024-11-11T16:15:11,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/93287a51ffdf4bbebf814bed4fb37a52 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/93287a51ffdf4bbebf814bed4fb37a52 2024-11-11T16:15:11,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ad6c1d31c60e484b9cd5c98808b34516 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ad6c1d31c60e484b9cd5c98808b34516 2024-11-11T16:15:11,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9735ef6fa40344af800a16153ab87f13 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9735ef6fa40344af800a16153ab87f13 2024-11-11T16:15:11,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/246d9935e50145149fe828698a8dcb13 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/246d9935e50145149fe828698a8dcb13 2024-11-11T16:15:11,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/0e5bd3f87caf4cacbe91bad5ddbc962e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/0e5bd3f87caf4cacbe91bad5ddbc962e 2024-11-11T16:15:11,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/79ed526be5264e019712423432a800b8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/79ed526be5264e019712423432a800b8 2024-11-11T16:15:11,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9ab6e899462c435face6bed486505fc7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/9ab6e899462c435face6bed486505fc7 2024-11-11T16:15:11,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/5ee6db9a6e8940909e910c6aea1714f6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/5ee6db9a6e8940909e910c6aea1714f6 2024-11-11T16:15:11,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e16e6fe0e5714745aa976a613dd086e9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e16e6fe0e5714745aa976a613dd086e9 2024-11-11T16:15:11,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/23a5f5825c3f4b9cbc5170f771d78a87 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/23a5f5825c3f4b9cbc5170f771d78a87 2024-11-11T16:15:11,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fe4c19cc0ef6493d9e0119b94ffe0e23 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/fe4c19cc0ef6493d9e0119b94ffe0e23 2024-11-11T16:15:11,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/509f152f81fd43f2856644f0f69d2b86 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/509f152f81fd43f2856644f0f69d2b86 2024-11-11T16:15:11,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e8d12e6f8d324933b72514ce4b0ef5a1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/e8d12e6f8d324933b72514ce4b0ef5a1 2024-11-11T16:15:11,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/bac9bf2bddff4a518bbacbe3da8c0aa7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/bac9bf2bddff4a518bbacbe3da8c0aa7 2024-11-11T16:15:11,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/a9ac08cfb582426baf40e18026e540fc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/a9ac08cfb582426baf40e18026e540fc 2024-11-11T16:15:11,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/2c62593f94964983bc81094b9761e7b6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/2c62593f94964983bc81094b9761e7b6 2024-11-11T16:15:11,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ce5147180f9f4107a0efe04df658f6e3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/ce5147180f9f4107a0efe04df658f6e3 2024-11-11T16:15:11,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/edd954bd4b7545368415b997577302f4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/03fa96aea33a4e18a2fd540512c5499c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ac6290ce478240ebb6f3335dd73332e6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a95a69a9aaf54b9f9dad9fd5fe0fc897, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/b30f7b4d84344a1db4b141277af32fb1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/1c87b0e18aca498798bba26a2b141a11, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0509b79519194e85953da9ecb295e99d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/992806e1f8f24ff8a9af739c77557ee0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0b113f82d3514d8a8a01463fe2d170bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d3f9bbb6d122478187325895684c7efb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/730a5e21a2724dc28b267ebd2d2a3e3c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/67f3abf54e95478e88713dd9045d17d2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/69c8faa69e11441f8ac2776ceaaaa158, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d9fe8834ed7d4a56aab56ae366132507, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a2715b99ec1643eb95d9f857fdaf20fb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/8126355a4f6446d091603ba3d227e7dc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/f6c475efc41343cdbc190552e895d0a0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/7301ae1019ff4be1a2e519b7afd1aa89, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/9db376d453ae495aa0a8b4f1e832a8bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a6f530b1ad514639852378dadebac8ba, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/bf599385ec9c43b282b880977ab43d03, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/43eac536e30d43aa98ee840162e2add5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/86a06d5d053f463cb359753458f5f3ae] to archive 2024-11-11T16:15:11,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:15:11,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/edd954bd4b7545368415b997577302f4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/edd954bd4b7545368415b997577302f4 2024-11-11T16:15:11,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/03fa96aea33a4e18a2fd540512c5499c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/03fa96aea33a4e18a2fd540512c5499c 2024-11-11T16:15:11,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ac6290ce478240ebb6f3335dd73332e6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ac6290ce478240ebb6f3335dd73332e6 2024-11-11T16:15:11,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a95a69a9aaf54b9f9dad9fd5fe0fc897 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a95a69a9aaf54b9f9dad9fd5fe0fc897 2024-11-11T16:15:11,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/b30f7b4d84344a1db4b141277af32fb1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/b30f7b4d84344a1db4b141277af32fb1 2024-11-11T16:15:11,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/1c87b0e18aca498798bba26a2b141a11 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/1c87b0e18aca498798bba26a2b141a11 2024-11-11T16:15:11,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0509b79519194e85953da9ecb295e99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0509b79519194e85953da9ecb295e99d 2024-11-11T16:15:11,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/992806e1f8f24ff8a9af739c77557ee0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/992806e1f8f24ff8a9af739c77557ee0 2024-11-11T16:15:11,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0b113f82d3514d8a8a01463fe2d170bb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/0b113f82d3514d8a8a01463fe2d170bb 2024-11-11T16:15:11,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d3f9bbb6d122478187325895684c7efb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d3f9bbb6d122478187325895684c7efb 2024-11-11T16:15:11,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/730a5e21a2724dc28b267ebd2d2a3e3c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/730a5e21a2724dc28b267ebd2d2a3e3c 2024-11-11T16:15:11,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/67f3abf54e95478e88713dd9045d17d2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/67f3abf54e95478e88713dd9045d17d2 2024-11-11T16:15:11,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/69c8faa69e11441f8ac2776ceaaaa158 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/69c8faa69e11441f8ac2776ceaaaa158 2024-11-11T16:15:11,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d9fe8834ed7d4a56aab56ae366132507 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/d9fe8834ed7d4a56aab56ae366132507 2024-11-11T16:15:11,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a2715b99ec1643eb95d9f857fdaf20fb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a2715b99ec1643eb95d9f857fdaf20fb 2024-11-11T16:15:11,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/8126355a4f6446d091603ba3d227e7dc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/8126355a4f6446d091603ba3d227e7dc 2024-11-11T16:15:11,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/f6c475efc41343cdbc190552e895d0a0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/f6c475efc41343cdbc190552e895d0a0 2024-11-11T16:15:11,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/7301ae1019ff4be1a2e519b7afd1aa89 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/7301ae1019ff4be1a2e519b7afd1aa89 2024-11-11T16:15:11,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/9db376d453ae495aa0a8b4f1e832a8bb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/9db376d453ae495aa0a8b4f1e832a8bb 2024-11-11T16:15:11,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a6f530b1ad514639852378dadebac8ba to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a6f530b1ad514639852378dadebac8ba 2024-11-11T16:15:11,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/bf599385ec9c43b282b880977ab43d03 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/bf599385ec9c43b282b880977ab43d03 2024-11-11T16:15:11,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/43eac536e30d43aa98ee840162e2add5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/43eac536e30d43aa98ee840162e2add5 2024-11-11T16:15:11,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/86a06d5d053f463cb359753458f5f3ae to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/86a06d5d053f463cb359753458f5f3ae 2024-11-11T16:15:11,533 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/recovered.edits/341.seqid, newMaxSeqId=341, maxSeqId=4 2024-11-11T16:15:11,534 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. 2024-11-11T16:15:11,534 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for c44034eee8380292fc203bc6a539d00c: Waiting for close lock at 1731341709981Running coprocessor pre-close hooks at 1731341709981Disabling compacts and flushes for region at 1731341709981Disabling writes for close at 1731341710851 (+870 ms)Obtaining lock to block concurrent updates at 1731341710851Preparing flush snapshotting stores in c44034eee8380292fc203bc6a539d00c at 1731341710851Finished memstore snapshotting TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731341710852 (+1 ms)Flushing stores of TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c. at 1731341710855 (+3 ms)Flushing c44034eee8380292fc203bc6a539d00c/A: creating writer at 1731341710855Flushing mob file c44034eee8380292fc203bc6a539d00c/A: appending metadata at 1731341710865 (+10 ms)Flushing mob file c44034eee8380292fc203bc6a539d00c/A: closing flushed file at 1731341710865Flushing c44034eee8380292fc203bc6a539d00c/A: appending metadata at 1731341710897 (+32 ms)Flushing c44034eee8380292fc203bc6a539d00c/A: closing flushed file at 1731341710898 (+1 ms)Flushing c44034eee8380292fc203bc6a539d00c/B: creating writer at 1731341711314 (+416 ms)Flushing c44034eee8380292fc203bc6a539d00c/B: appending metadata at 1731341711318 (+4 ms)Flushing c44034eee8380292fc203bc6a539d00c/B: closing flushed file at 1731341711318Flushing c44034eee8380292fc203bc6a539d00c/C: creating writer at 1731341711345 (+27 ms)Flushing c44034eee8380292fc203bc6a539d00c/C: appending metadata at 1731341711349 (+4 ms)Flushing c44034eee8380292fc203bc6a539d00c/C: closing flushed file at 1731341711349Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18b26a47: reopening flushed file at 1731341711375 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f6557c: reopening flushed file at 1731341711382 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ff85efb: reopening flushed file at 1731341711390 (+8 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c44034eee8380292fc203bc6a539d00c in 548ms, sequenceid=338, compaction requested=true at 1731341711399 (+9 ms)Writing region close event to WAL at 1731341711530 (+131 ms)Running coprocessor post-close hooks at 1731341711534 (+4 ms)Closed at 1731341711534 2024-11-11T16:15:11,536 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:11,536 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=c44034eee8380292fc203bc6a539d00c, regionState=CLOSED 2024-11-11T16:15:11,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:15:11,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T16:15:11,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure c44034eee8380292fc203bc6a539d00c, server=16b413a53992,45121,1731341648557 in 1.7130 sec 2024-11-11T16:15:11,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-11T16:15:11,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c44034eee8380292fc203bc6a539d00c, UNASSIGN in 1.7190 sec 2024-11-11T16:15:11,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T16:15:11,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7240 sec 2024-11-11T16:15:11,544 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341711544"}]},"ts":"1731341711544"} 2024-11-11T16:15:11,545 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T16:15:11,546 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T16:15:11,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7410 sec 2024-11-11T16:15:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T16:15:11,967 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:11,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T16:15:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:11,969 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T16:15:11,970 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:11,973 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:11,977 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/recovered.edits] 2024-11-11T16:15:11,984 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/14fc1f1d4ff94aee95c86721386e5c96 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/14fc1f1d4ff94aee95c86721386e5c96 2024-11-11T16:15:11,985 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6f5c9e46318b462bb0a11564813852dd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/6f5c9e46318b462bb0a11564813852dd 2024-11-11T16:15:11,987 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/cf2e5608efca4d21bdfdb23b3292240e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/A/cf2e5608efca4d21bdfdb23b3292240e 2024-11-11T16:15:11,990 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/1353c174abb54c5296a0cc8a3661f279 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/1353c174abb54c5296a0cc8a3661f279 2024-11-11T16:15:11,991 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/405f8175f4774854a1d06060f738831d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/405f8175f4774854a1d06060f738831d 2024-11-11T16:15:11,992 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/6f3867ba07ff4d00b231fe055a865412 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/B/6f3867ba07ff4d00b231fe055a865412 2024-11-11T16:15:11,995 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/73cf28f0908b4d09978f6c2fb69036b7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/73cf28f0908b4d09978f6c2fb69036b7 2024-11-11T16:15:11,997 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a065aec424c34cd59a6b2ec84eff436b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/a065aec424c34cd59a6b2ec84eff436b 2024-11-11T16:15:11,998 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ab17b58ed0974192aadc8d84ee2eff7e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/C/ab17b58ed0974192aadc8d84ee2eff7e 2024-11-11T16:15:12,001 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/recovered.edits/341.seqid to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c/recovered.edits/341.seqid 2024-11-11T16:15:12,001 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,001 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T16:15:12,002 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T16:15:12,003 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T16:15:12,008 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111063bf4ab9f814d8d9e5b8dc4c69e7eac_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111063bf4ab9f814d8d9e5b8dc4c69e7eac_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,009 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112c7a909f13024653a37e1b6f2db13dee_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112c7a909f13024653a37e1b6f2db13dee_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,010 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112ce8dc44299c4eacafc0aef08c992303_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112ce8dc44299c4eacafc0aef08c992303_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,011 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112dfa717d542a4874b53fd9369d7c5b58_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112dfa717d542a4874b53fd9369d7c5b58_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,012 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b6ef6c0bcc44972b0c463b9f043eac1_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b6ef6c0bcc44972b0c463b9f043eac1_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,013 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113c46d956ef5a4e93aca9d33edcc67183_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113c46d956ef5a4e93aca9d33edcc67183_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,014 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114ba517f47842434c8010b2f8533b4e22_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114ba517f47842434c8010b2f8533b4e22_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,016 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156e5b08ad86e475c838f8d2d437a880f_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156e5b08ad86e475c838f8d2d437a880f_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,017 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169a86ded5adf46f5ab9210a311b33f23_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169a86ded5adf46f5ab9210a311b33f23_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,018 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116f1b6e84c0db4e54800df39f2ec8ab45_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116f1b6e84c0db4e54800df39f2ec8ab45_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,019 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111172df837c7de54c429fec5be71f296bfb_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111172df837c7de54c429fec5be71f296bfb_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,020 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a07cc95e266248bf82a005ea9de537fa_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a07cc95e266248bf82a005ea9de537fa_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,022 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be14bfcd09ae40d39a6d11164f86646d_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be14bfcd09ae40d39a6d11164f86646d_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,023 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e2e26ac4cbf54c928ee94a98cda87eac_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e2e26ac4cbf54c928ee94a98cda87eac_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,024 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebe5805a65e74fd49a00d81ce5856e11_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebe5805a65e74fd49a00d81ce5856e11_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,025 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f3847769d32346fd9c112ee89f8ba7c1_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f3847769d32346fd9c112ee89f8ba7c1_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,027 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f468bb4d88554ef99d4307dbea7bdd04_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f468bb4d88554ef99d4307dbea7bdd04_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,028 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f75d39c2da9a4116a7a4d79160bb46b7_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f75d39c2da9a4116a7a4d79160bb46b7_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,029 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8e3329572314d6dbf65b1a90a363743_c44034eee8380292fc203bc6a539d00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8e3329572314d6dbf65b1a90a363743_c44034eee8380292fc203bc6a539d00c 2024-11-11T16:15:12,030 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T16:15:12,032 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:12,036 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T16:15:12,039 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T16:15:12,040 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:12,040 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T16:15:12,041 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731341712040"}]},"ts":"9223372036854775807"} 2024-11-11T16:15:12,043 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T16:15:12,043 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c44034eee8380292fc203bc6a539d00c, NAME => 'TestAcidGuarantees,,1731341684600.c44034eee8380292fc203bc6a539d00c.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T16:15:12,043 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T16:15:12,044 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731341712043"}]},"ts":"9223372036854775807"} 2024-11-11T16:15:12,045 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T16:15:12,046 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:12,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-11T16:15:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T16:15:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T16:15:12,077 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:12,093 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 243) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1554041885_22 at /127.0.0.1:56964 [Waiting for operation #417] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1554041885_22 at /127.0.0.1:57100 [Waiting for operation #387] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1306082440_22 at /127.0.0.1:39386 [Waiting for operation #459] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1306082440_22 at /127.0.0.1:56960 [Waiting for operation #434] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 651) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1268 (was 2467) 2024-11-11T16:15:12,104 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=1267 2024-11-11T16:15:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:15:12,106 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:15:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:12,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T16:15:12,108 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:12,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-11T16:15:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T16:15:12,110 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T16:15:12,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742058_1234 (size=1061) 2024-11-11T16:15:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T16:15:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T16:15:12,518 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:15:12,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742059_1235 (size=53) 2024-11-11T16:15:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T16:15:12,924 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:12,924 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8a30e5caa9f1f33bab5e05117c34d967, disabling compactions & flushes 2024-11-11T16:15:12,924 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:12,924 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:12,925 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. after waiting 0 ms 2024-11-11T16:15:12,925 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:12,925 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:12,925 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a30e5caa9f1f33bab5e05117c34d967: Waiting for close lock at 1731341712924Disabling compacts and flushes for region at 1731341712924Disabling writes for close at 1731341712925 (+1 ms)Writing region close event to WAL at 1731341712925Closed at 1731341712925 2024-11-11T16:15:12,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T16:15:12,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731341712926"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341712926"}]},"ts":"1731341712926"} 2024-11-11T16:15:12,928 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T16:15:12,929 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T16:15:12,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341712929"}]},"ts":"1731341712929"} 2024-11-11T16:15:12,931 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T16:15:12,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, ASSIGN}] 2024-11-11T16:15:12,933 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, ASSIGN 2024-11-11T16:15:12,934 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, ASSIGN; state=OFFLINE, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=false 2024-11-11T16:15:13,085 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=8a30e5caa9f1f33bab5e05117c34d967, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:13,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, ASSIGN because future has completed 2024-11-11T16:15:13,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:15:13,243 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:13,243 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:15:13,244 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,244 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:13,244 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,244 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,245 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,247 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:13,247 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a30e5caa9f1f33bab5e05117c34d967 columnFamilyName A 2024-11-11T16:15:13,247 DEBUG [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:13,247 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.HStore(327): Store=8a30e5caa9f1f33bab5e05117c34d967/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:13,247 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,249 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:13,249 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a30e5caa9f1f33bab5e05117c34d967 columnFamilyName B 2024-11-11T16:15:13,249 DEBUG [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:13,250 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.HStore(327): Store=8a30e5caa9f1f33bab5e05117c34d967/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:13,250 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,251 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:13,251 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a30e5caa9f1f33bab5e05117c34d967 columnFamilyName C 2024-11-11T16:15:13,251 DEBUG [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:13,251 INFO [StoreOpener-8a30e5caa9f1f33bab5e05117c34d967-1 {}] regionserver.HStore(327): Store=8a30e5caa9f1f33bab5e05117c34d967/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:13,252 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:13,252 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,252 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,253 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,254 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,254 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,254 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:15:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T16:15:13,255 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,258 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:13,258 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 8a30e5caa9f1f33bab5e05117c34d967; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75332641, jitterRate=0.1225438266992569}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:15:13,258 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:13,259 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 8a30e5caa9f1f33bab5e05117c34d967: Running coprocessor pre-open hook at 1731341713244Writing region info on filesystem at 1731341713244Initializing all the Stores at 1731341713245 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341713245Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341713245Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341713245Cleaning up temporary data from old regions at 1731341713254 (+9 ms)Running coprocessor post-open hooks at 1731341713258 (+4 ms)Region opened successfully at 1731341713259 (+1 ms) 2024-11-11T16:15:13,260 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., pid=59, masterSystemTime=1731341713240 2024-11-11T16:15:13,262 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:13,262 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:13,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=8a30e5caa9f1f33bab5e05117c34d967, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:13,264 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33601 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=16b413a53992,45121,1731341648557, table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T16:15:13,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:15:13,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T16:15:13,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 in 179 msec 2024-11-11T16:15:13,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-11T16:15:13,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, ASSIGN in 337 msec 2024-11-11T16:15:13,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T16:15:13,271 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341713271"}]},"ts":"1731341713271"} 2024-11-11T16:15:13,273 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T16:15:13,274 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T16:15:13,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-11T16:15:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T16:15:14,266 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:14,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159b31ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,269 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21fed69e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,271 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f3ba15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,272 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,275 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:15:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,33601,1731341647570 2024-11-11T16:15:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28a8bf55 2024-11-11T16:15:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:15:14,277 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:15:14,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0ff7e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5381bc7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,281 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171abfe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563a4482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,284 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388b265e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@430719b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d77d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c42f524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4965c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef78528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a55f402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5b65f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e69269f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,301 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e7ebb33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a63f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1c9fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399b4bff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e720638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2984ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,313 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3007885f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f832ccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558a1543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,317 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc2ed8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,318 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41adfd91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd64e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,320 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d67a3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1060c4fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3faa7931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:14,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:14,325 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@702384f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:14,327 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29e59a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:14,331 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,334 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,351 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-11T16:15:14,364 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:14,364 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:14,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T16:15:14,367 DEBUG [Thread-1116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,378 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,393 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,395 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,406 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:14,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:14,417 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:14,433 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:14,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:14,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:15:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T16:15:14,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341774470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/ad7f5ce26100423e90f6e31b4a5cb6a2 is 50, key is test_row_0/A:col10/1731341714416/Put/seqid=0 2024-11-11T16:15:14,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341774473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341774474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341774476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341774479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742060_1236 (size=12001) 2024-11-11T16:15:14,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341774585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341774585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341774596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341774596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341774597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:14,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:14,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T16:15:14,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341774796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341774796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341774806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341774806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341774806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:14,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:14,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:14,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/ad7f5ce26100423e90f6e31b4a5cb6a2 2024-11-11T16:15:14,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:14,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T16:15:15,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/aebe38170a354a998d5d58732b2b4a09 is 50, key is test_row_0/B:col10/1731341714416/Put/seqid=0 2024-11-11T16:15:15,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742061_1237 (size=12001) 2024-11-11T16:15:15,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341775106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341775112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341775116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341775116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341775116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/aebe38170a354a998d5d58732b2b4a09 2024-11-11T16:15:15,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:15,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:15,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T16:15:15,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/2bd21fb4684848989a7c73be5032db05 is 50, key is test_row_0/C:col10/1731341714416/Put/seqid=0 2024-11-11T16:15:15,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742062_1238 (size=12001) 2024-11-11T16:15:15,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:15,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:15,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341775626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341775627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341775629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341775629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341775636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:15,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:15,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:15,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:15,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:15,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/2bd21fb4684848989a7c73be5032db05 2024-11-11T16:15:15,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/ad7f5ce26100423e90f6e31b4a5cb6a2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ad7f5ce26100423e90f6e31b4a5cb6a2 2024-11-11T16:15:15,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ad7f5ce26100423e90f6e31b4a5cb6a2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T16:15:15,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/aebe38170a354a998d5d58732b2b4a09 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/aebe38170a354a998d5d58732b2b4a09 2024-11-11T16:15:15,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/aebe38170a354a998d5d58732b2b4a09, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T16:15:15,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/2bd21fb4684848989a7c73be5032db05 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/2bd21fb4684848989a7c73be5032db05 2024-11-11T16:15:15,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/2bd21fb4684848989a7c73be5032db05, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T16:15:15,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a30e5caa9f1f33bab5e05117c34d967 in 1546ms, sequenceid=13, compaction requested=false 2024-11-11T16:15:15,981 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T16:15:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:16,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T16:15:16,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:16,075 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:15:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c412a9a941744691bf641da5df604ec6 is 50, key is test_row_0/A:col10/1731341714473/Put/seqid=0 2024-11-11T16:15:16,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742063_1239 (size=12001) 2024-11-11T16:15:16,525 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c412a9a941744691bf641da5df604ec6 2024-11-11T16:15:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T16:15:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/84459d028c374ef5af936eea1e33f9d0 is 50, key is test_row_0/B:col10/1731341714473/Put/seqid=0 2024-11-11T16:15:16,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742064_1240 (size=12001) 2024-11-11T16:15:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341776668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341776671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341776675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341776676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341776677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341776786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341776796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341776796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341776798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341776798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:16,979 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/84459d028c374ef5af936eea1e33f9d0 2024-11-11T16:15:16,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e9b921a4a8a44865a587a6fbffc3642d is 50, key is test_row_0/C:col10/1731341714473/Put/seqid=0 2024-11-11T16:15:16,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341776995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341777007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341777007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341777008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341777009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742065_1241 (size=12001) 2024-11-11T16:15:17,033 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e9b921a4a8a44865a587a6fbffc3642d 2024-11-11T16:15:17,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c412a9a941744691bf641da5df604ec6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c412a9a941744691bf641da5df604ec6 2024-11-11T16:15:17,046 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c412a9a941744691bf641da5df604ec6, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T16:15:17,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/84459d028c374ef5af936eea1e33f9d0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/84459d028c374ef5af936eea1e33f9d0 2024-11-11T16:15:17,057 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/84459d028c374ef5af936eea1e33f9d0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T16:15:17,059 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T16:15:17,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e9b921a4a8a44865a587a6fbffc3642d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e9b921a4a8a44865a587a6fbffc3642d 2024-11-11T16:15:17,089 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e9b921a4a8a44865a587a6fbffc3642d, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T16:15:17,097 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8a30e5caa9f1f33bab5e05117c34d967 in 1021ms, sequenceid=33, compaction requested=false 2024-11-11T16:15:17,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-11T16:15:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-11T16:15:17,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T16:15:17,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7500 sec 2024-11-11T16:15:17,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.7580 sec 2024-11-11T16:15:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:17,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:15:17,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:17,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:17,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:17,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:17,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:17,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:17,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/43834f3144b94d4e87e39609a042df87 is 50, key is test_row_0/A:col10/1731341716672/Put/seqid=0 2024-11-11T16:15:17,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742066_1242 (size=12001) 2024-11-11T16:15:17,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/43834f3144b94d4e87e39609a042df87 2024-11-11T16:15:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341777355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341777358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341777358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341777359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341777359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/9cdbc7c5fb454e74a55459cd79da0651 is 50, key is test_row_0/B:col10/1731341716672/Put/seqid=0 2024-11-11T16:15:17,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742067_1243 (size=12001) 2024-11-11T16:15:17,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/9cdbc7c5fb454e74a55459cd79da0651 2024-11-11T16:15:17,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca is 50, key is test_row_0/C:col10/1731341716672/Put/seqid=0 2024-11-11T16:15:17,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341777476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341777476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341777477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341777477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341777477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742068_1244 (size=12001) 2024-11-11T16:15:17,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca 2024-11-11T16:15:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/43834f3144b94d4e87e39609a042df87 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/43834f3144b94d4e87e39609a042df87 2024-11-11T16:15:17,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/43834f3144b94d4e87e39609a042df87, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T16:15:17,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/9cdbc7c5fb454e74a55459cd79da0651 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/9cdbc7c5fb454e74a55459cd79da0651 2024-11-11T16:15:17,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/9cdbc7c5fb454e74a55459cd79da0651, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T16:15:17,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca 2024-11-11T16:15:17,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T16:15:17,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8a30e5caa9f1f33bab5e05117c34d967 in 297ms, sequenceid=46, compaction requested=true 2024-11-11T16:15:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:15:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T16:15:17,612 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:17,613 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:17,615 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:17,615 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:17,615 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:17,615 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ad7f5ce26100423e90f6e31b4a5cb6a2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c412a9a941744691bf641da5df604ec6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/43834f3144b94d4e87e39609a042df87] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=35.2 K 2024-11-11T16:15:17,617 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7f5ce26100423e90f6e31b4a5cb6a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731341714355 2024-11-11T16:15:17,617 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c412a9a941744691bf641da5df604ec6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731341714467 2024-11-11T16:15:17,618 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43834f3144b94d4e87e39609a042df87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731341716672 2024-11-11T16:15:17,627 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:17,627 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:17,627 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:17,628 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/2bd21fb4684848989a7c73be5032db05, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e9b921a4a8a44865a587a6fbffc3642d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=35.2 K 2024-11-11T16:15:17,629 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd21fb4684848989a7c73be5032db05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731341714355 2024-11-11T16:15:17,629 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b921a4a8a44865a587a6fbffc3642d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731341714467 2024-11-11T16:15:17,630 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 32c5e7ba2c7d4509bc74ca4dbcbe1dca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731341716672 2024-11-11T16:15:17,639 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:17,640 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/9b4ec9727ccd4a8da443724fe16cbd45 is 50, key is test_row_0/A:col10/1731341716672/Put/seqid=0 2024-11-11T16:15:17,651 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:17,651 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/28871775dd664610bc869788d917d2c0 is 50, key is test_row_0/C:col10/1731341716672/Put/seqid=0 2024-11-11T16:15:17,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742069_1245 (size=12104) 2024-11-11T16:15:17,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341717351, max timestamp=1731341717689Num uniques 859; 2024-11-11T16:15:17,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341717351, max timestamp=1731341717689Num uniques 859; 2024-11-11T16:15:17,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:17,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:17,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-11T16:15:17,695 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/9b4ec9727ccd4a8da443724fe16cbd45 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/9b4ec9727ccd4a8da443724fe16cbd45 2024-11-11T16:15:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341717351, max timestamp=1731341717689Num uniques 859; 2024-11-11T16:15:17,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:17,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:17,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:17,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742070_1246 (size=12104) 2024-11-11T16:15:17,706 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into 9b4ec9727ccd4a8da443724fe16cbd45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:17,706 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:17,706 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341717609; duration=0sec 2024-11-11T16:15:17,706 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:17,707 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:17,707 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:17,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/584b29ce8376499dbf8eec3cecbdb82a is 50, key is test_row_0/A:col10/1731341717353/Put/seqid=0 2024-11-11T16:15:17,711 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:17,711 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:17,711 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:17,712 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/aebe38170a354a998d5d58732b2b4a09, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/84459d028c374ef5af936eea1e33f9d0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/9cdbc7c5fb454e74a55459cd79da0651] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=35.2 K 2024-11-11T16:15:17,713 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting aebe38170a354a998d5d58732b2b4a09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731341714355 2024-11-11T16:15:17,713 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84459d028c374ef5af936eea1e33f9d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731341714467 2024-11-11T16:15:17,714 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cdbc7c5fb454e74a55459cd79da0651, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731341716672 2024-11-11T16:15:17,718 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/28871775dd664610bc869788d917d2c0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/28871775dd664610bc869788d917d2c0 2024-11-11T16:15:17,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742071_1247 (size=12001) 2024-11-11T16:15:17,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/584b29ce8376499dbf8eec3cecbdb82a 2024-11-11T16:15:17,734 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#204 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:17,734 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b3d86e1cf8024a6f81f48bf0fd1b8eb7 is 50, key is test_row_0/B:col10/1731341716672/Put/seqid=0 2024-11-11T16:15:17,739 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into 28871775dd664610bc869788d917d2c0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:17,739 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:17,739 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341717609; duration=0sec 2024-11-11T16:15:17,740 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:17,740 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:17,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/018144469a0c4eb09882a47a1ab77d2e is 50, key is test_row_0/B:col10/1731341717353/Put/seqid=0 2024-11-11T16:15:17,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341717723, max timestamp=1731341717742Num uniques 859; 2024-11-11T16:15:17,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341717723, max timestamp=1731341717742Num uniques 859; 2024-11-11T16:15:17,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:17,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:17,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341717723, max timestamp=1731341717742Num uniques 859; 2024-11-11T16:15:17,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:17,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742072_1248 (size=12104) 2024-11-11T16:15:17,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742073_1249 (size=12001) 2024-11-11T16:15:17,784 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b3d86e1cf8024a6f81f48bf0fd1b8eb7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b3d86e1cf8024a6f81f48bf0fd1b8eb7 2024-11-11T16:15:17,791 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into b3d86e1cf8024a6f81f48bf0fd1b8eb7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:17,792 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:17,792 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341717609; duration=0sec 2024-11-11T16:15:17,792 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:17,792 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:17,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341777852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:17,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341717800, max timestamp=1731341717862Num uniques 1118; 2024-11-11T16:15:17,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341717800, max timestamp=1731341717862Num uniques 1118; 2024-11-11T16:15:17,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-11-11T16:15:17,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-11-11T16:15:17,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341717800, max timestamp=1731341717862Num uniques 1118; 2024-11-11T16:15:17,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-11-11T16:15:17,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341717918, max timestamp=1731341717964Num uniques 1009; 2024-11-11T16:15:17,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-11T16:15:17,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341717918, max timestamp=1731341717964Num uniques 1009; 2024-11-11T16:15:17,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-11T16:15:17,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341717918, max timestamp=1731341717964Num uniques 1009; 2024-11-11T16:15:17,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-11T16:15:18,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341718011, max timestamp=1731341718030Num uniques 1086; 2024-11-11T16:15:18,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341718011, max timestamp=1731341718030Num uniques 1086; 2024-11-11T16:15:18,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-11T16:15:18,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-11T16:15:18,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341718011, max timestamp=1731341718030Num uniques 1086; 2024-11-11T16:15:18,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-11T16:15:18,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341718082, max timestamp=1731341718133Num uniques 1045; 2024-11-11T16:15:18,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T16:15:18,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341718082, max timestamp=1731341718133Num uniques 1045; 2024-11-11T16:15:18,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T16:15:18,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341718082, max timestamp=1731341718133Num uniques 1045; 2024-11-11T16:15:18,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-11T16:15:18,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/018144469a0c4eb09882a47a1ab77d2e 2024-11-11T16:15:18,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341718188, max timestamp=1731341718195Num uniques 1104; 2024-11-11T16:15:18,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-11T16:15:18,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341718188, max timestamp=1731341718195Num uniques 1009; 2024-11-11T16:15:18,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=7050, suffix heap size=132904,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=154 2024-11-11T16:15:18,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341718188, max timestamp=1731341718195Num uniques 1104; 2024-11-11T16:15:18,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-11T16:15:18,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/3ba349969d794acc9ed646ba7c2665a4 is 50, key is test_row_0/C:col10/1731341717353/Put/seqid=0 2024-11-11T16:15:18,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731341718236, max timestamp=1731341718250Num uniques 1013; 2024-11-11T16:15:18,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T16:15:18,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731341718236, max timestamp=1731341718250Num uniques 1013; 2024-11-11T16:15:18,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T16:15:18,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731341718236, max timestamp=1731341718250Num uniques 1013; 2024-11-11T16:15:18,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T16:15:18,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742074_1250 (size=12001) 2024-11-11T16:15:18,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731341718279, max timestamp=1731341718296Num uniques 1022; 2024-11-11T16:15:18,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-11T16:15:18,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731341718279, max timestamp=1731341718297Num uniques 1022; 2024-11-11T16:15:18,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-11T16:15:18,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731341718279, max timestamp=1731341718297Num uniques 1022; 2024-11-11T16:15:18,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-11T16:15:18,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731341718311, max timestamp=1731341718339Num uniques 1031; 2024-11-11T16:15:18,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-11T16:15:18,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731341718311, max timestamp=1731341718339Num uniques 1031; 2024-11-11T16:15:18,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-11T16:15:18,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731341718311, max timestamp=1731341718340Num uniques 1031; 2024-11-11T16:15:18,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-11T16:15:18,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=247, cellSize=11312, totalHeapSize=25928, min timestamp=1731341718350, max timestamp=1731341718377Num uniques 1040; 2024-11-11T16:15:18,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=11312, suffix heap size=136128,new segment heap size=25928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=247 2024-11-11T16:15:18,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731341718350, max timestamp=1731341718377Num uniques 1040; 2024-11-11T16:15:18,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-11T16:15:18,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731341718350, max timestamp=1731341718377Num uniques 1040; 2024-11-11T16:15:18,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-11T16:15:18,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341778442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341778447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341778450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731341718434, max timestamp=1731341718449Num uniques 1106; 2024-11-11T16:15:18,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50656, new segment data size=11404, suffix heap size=142992,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1106, new segment cells count=249 2024-11-11T16:15:18,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11394, totalHeapSize=26136, min timestamp=1731341718434, max timestamp=1731341718450Num uniques 1099; 2024-11-11T16:15:18,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=11394, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-11T16:15:18,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11394, totalHeapSize=26136, min timestamp=1731341718434, max timestamp=1731341718450Num uniques 1099; 2024-11-11T16:15:18,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=11394, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-11T16:15:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731341718495, max timestamp=1731341718524Num uniques 1108; 2024-11-11T16:15:18,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731341718495, max timestamp=1731341718524Num uniques 1108; 2024-11-11T16:15:18,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731341718495, max timestamp=1731341718524Num uniques 1108; 2024-11-11T16:15:18,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-11T16:15:18,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-11T16:15:18,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-11T16:15:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T16:15:18,546 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:18,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-11T16:15:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T16:15:18,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:18,565 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:18,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:18,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731341718575, max timestamp=1731341718590Num uniques 1017; 2024-11-11T16:15:18,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-11T16:15:18,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731341718575, max timestamp=1731341718590Num uniques 1017; 2024-11-11T16:15:18,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-11T16:15:18,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731341718581, max timestamp=1731341718591Num uniques 1017; 2024-11-11T16:15:18,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-11T16:15:18,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731341718626, max timestamp=1731341718646Num uniques 1126; 2024-11-11T16:15:18,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=10346, suffix heap size=145072,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=226 2024-11-11T16:15:18,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731341718626, max timestamp=1731341718646Num uniques 1126; 2024-11-11T16:15:18,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731341718626, max timestamp=1731341718646Num uniques 1026; 2024-11-11T16:15:18,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-11T16:15:18,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=10346, suffix heap size=145072,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=226 2024-11-11T16:15:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T16:15:18,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/3ba349969d794acc9ed646ba7c2665a4 2024-11-11T16:15:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/584b29ce8376499dbf8eec3cecbdb82a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/584b29ce8376499dbf8eec3cecbdb82a 2024-11-11T16:15:18,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/584b29ce8376499dbf8eec3cecbdb82a, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T16:15:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/018144469a0c4eb09882a47a1ab77d2e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/018144469a0c4eb09882a47a1ab77d2e 2024-11-11T16:15:18,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/018144469a0c4eb09882a47a1ab77d2e, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T16:15:18,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/3ba349969d794acc9ed646ba7c2665a4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3ba349969d794acc9ed646ba7c2665a4 2024-11-11T16:15:18,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3ba349969d794acc9ed646ba7c2665a4, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T16:15:18,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=114.05 KB/116790 for 8a30e5caa9f1f33bab5e05117c34d967 in 1002ms, sequenceid=68, compaction requested=false 2024-11-11T16:15:18,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:18,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=134.18 KB heapSize=378.70 KB 2024-11-11T16:15:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:18,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T16:15:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:18,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:18,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:18,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:18,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c500ffa8b7e34fc8866fceb143cdba67 is 50, key is test_row_0/A:col10/1731341718699/Put/seqid=0 2024-11-11T16:15:18,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742075_1251 (size=12301) 2024-11-11T16:15:18,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c500ffa8b7e34fc8866fceb143cdba67 2024-11-11T16:15:18,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341778730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341778733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341778734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341778742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341778743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/0fc5b7336e244f538cea735e147184a6 is 50, key is test_row_0/B:col10/1731341718699/Put/seqid=0 2024-11-11T16:15:18,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742076_1252 (size=12301) 2024-11-11T16:15:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341778846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341778847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341778850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341778856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341778856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T16:15:18,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T16:15:18,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:18,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:18,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:18,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T16:15:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341779056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341779056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341779056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341779066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341779066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/0fc5b7336e244f538cea735e147184a6 2024-11-11T16:15:19,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T16:15:19,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T16:15:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/5299e2d34acf46a68ba2d8e23d59cfc5 is 50, key is test_row_0/C:col10/1731341718699/Put/seqid=0 2024-11-11T16:15:19,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742077_1253 (size=12301) 2024-11-11T16:15:19,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/5299e2d34acf46a68ba2d8e23d59cfc5 2024-11-11T16:15:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c500ffa8b7e34fc8866fceb143cdba67 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c500ffa8b7e34fc8866fceb143cdba67 2024-11-11T16:15:19,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c500ffa8b7e34fc8866fceb143cdba67, entries=150, sequenceid=331, filesize=12.0 K 2024-11-11T16:15:19,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/0fc5b7336e244f538cea735e147184a6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0fc5b7336e244f538cea735e147184a6 2024-11-11T16:15:19,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0fc5b7336e244f538cea735e147184a6, entries=150, sequenceid=331, filesize=12.0 K 2024-11-11T16:15:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/5299e2d34acf46a68ba2d8e23d59cfc5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5299e2d34acf46a68ba2d8e23d59cfc5 2024-11-11T16:15:19,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5299e2d34acf46a68ba2d8e23d59cfc5, entries=150, sequenceid=331, filesize=12.0 K 2024-11-11T16:15:19,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~398.58 KB/408144, currentSize=46.96 KB/48090 for 8a30e5caa9f1f33bab5e05117c34d967 in 565ms, sequenceid=331, compaction requested=true 2024-11-11T16:15:19,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:19,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:19,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:19,270 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:19,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:15:19,271 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:19,272 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:19,272 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:19,272 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,272 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b3d86e1cf8024a6f81f48bf0fd1b8eb7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/018144469a0c4eb09882a47a1ab77d2e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0fc5b7336e244f538cea735e147184a6] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=35.6 K 2024-11-11T16:15:19,273 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:19,273 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:19,273 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,273 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/9b4ec9727ccd4a8da443724fe16cbd45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/584b29ce8376499dbf8eec3cecbdb82a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c500ffa8b7e34fc8866fceb143cdba67] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=35.6 K 2024-11-11T16:15:19,273 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d86e1cf8024a6f81f48bf0fd1b8eb7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731341716672 2024-11-11T16:15:19,273 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4ec9727ccd4a8da443724fe16cbd45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731341716672 2024-11-11T16:15:19,274 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584b29ce8376499dbf8eec3cecbdb82a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731341717353 2024-11-11T16:15:19,274 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 018144469a0c4eb09882a47a1ab77d2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731341717353 2024-11-11T16:15:19,275 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c500ffa8b7e34fc8866fceb143cdba67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731341718699 2024-11-11T16:15:19,275 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc5b7336e244f538cea735e147184a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731341718699 2024-11-11T16:15:19,306 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:19,307 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/42ff5a7e4b8d463bb6bce1ce4842d5ad is 50, key is test_row_0/A:col10/1731341718699/Put/seqid=0 2024-11-11T16:15:19,326 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:19,326 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/0db7166dc35548039e4b7e77a48f0c90 is 50, key is test_row_0/B:col10/1731341718699/Put/seqid=0 2024-11-11T16:15:19,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T16:15:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742078_1254 (size=12507) 2024-11-11T16:15:19,352 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:19,356 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/42ff5a7e4b8d463bb6bce1ce4842d5ad as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/42ff5a7e4b8d463bb6bce1ce4842d5ad 2024-11-11T16:15:19,366 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into 42ff5a7e4b8d463bb6bce1ce4842d5ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:19,366 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:19,366 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341719270; duration=0sec 2024-11-11T16:15:19,366 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:19,366 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:19,366 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:19,368 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:19,368 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:19,369 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,369 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/28871775dd664610bc869788d917d2c0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3ba349969d794acc9ed646ba7c2665a4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5299e2d34acf46a68ba2d8e23d59cfc5] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=35.6 K 2024-11-11T16:15:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:19,371 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28871775dd664610bc869788d917d2c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731341716672 2024-11-11T16:15:19,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:19,371 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba349969d794acc9ed646ba7c2665a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731341717353 2024-11-11T16:15:19,372 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5299e2d34acf46a68ba2d8e23d59cfc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731341718699 2024-11-11T16:15:19,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/7b3bc2ff42384ad0bf6acdf3e55d9679 is 50, key is test_row_0/A:col10/1731341718720/Put/seqid=0 2024-11-11T16:15:19,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742079_1255 (size=12507) 2024-11-11T16:15:19,388 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/0db7166dc35548039e4b7e77a48f0c90 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0db7166dc35548039e4b7e77a48f0c90 2024-11-11T16:15:19,395 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into 0db7166dc35548039e4b7e77a48f0c90(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:19,395 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:19,395 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341719270; duration=0sec 2024-11-11T16:15:19,395 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:19,395 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:19,399 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:19,400 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/bcde75f8dc8c4331a06737b7c818c986 is 50, key is test_row_0/C:col10/1731341718699/Put/seqid=0 2024-11-11T16:15:19,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341719407, max timestamp=1731341719419Num uniques 859; 2024-11-11T16:15:19,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:19,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341719407, max timestamp=1731341719419Num uniques 859; 2024-11-11T16:15:19,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341719407, max timestamp=1731341719419Num uniques 859; 2024-11-11T16:15:19,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:19,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742080_1256 (size=12301) 2024-11-11T16:15:19,435 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/7b3bc2ff42384ad0bf6acdf3e55d9679 2024-11-11T16:15:19,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742081_1257 (size=12507) 2024-11-11T16:15:19,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/d93ce57d9ab847aabcdbf16297eb577a is 50, key is test_row_0/B:col10/1731341718720/Put/seqid=0 2024-11-11T16:15:19,455 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/bcde75f8dc8c4331a06737b7c818c986 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/bcde75f8dc8c4331a06737b7c818c986 2024-11-11T16:15:19,463 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into bcde75f8dc8c4331a06737b7c818c986(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:19,463 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:19,463 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341719271; duration=0sec 2024-11-11T16:15:19,463 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:19,463 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:19,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341779464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341779467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341779468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341779470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341779470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742082_1258 (size=12301) 2024-11-11T16:15:19,504 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/d93ce57d9ab847aabcdbf16297eb577a 2024-11-11T16:15:19,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/0607b13cc08741e48c3547457ddcdec7 is 50, key is test_row_0/C:col10/1731341718720/Put/seqid=0 2024-11-11T16:15:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341779586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341779586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341779587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341779587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742083_1259 (size=12301) 2024-11-11T16:15:19,599 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/0607b13cc08741e48c3547457ddcdec7 2024-11-11T16:15:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341779595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/7b3bc2ff42384ad0bf6acdf3e55d9679 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/7b3bc2ff42384ad0bf6acdf3e55d9679 2024-11-11T16:15:19,610 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/7b3bc2ff42384ad0bf6acdf3e55d9679, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T16:15:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/d93ce57d9ab847aabcdbf16297eb577a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d93ce57d9ab847aabcdbf16297eb577a 2024-11-11T16:15:19,617 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d93ce57d9ab847aabcdbf16297eb577a, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T16:15:19,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/0607b13cc08741e48c3547457ddcdec7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/0607b13cc08741e48c3547457ddcdec7 2024-11-11T16:15:19,624 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/0607b13cc08741e48c3547457ddcdec7, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T16:15:19,626 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 8a30e5caa9f1f33bab5e05117c34d967 in 286ms, sequenceid=341, compaction requested=false 2024-11-11T16:15:19,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:19,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T16:15:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T16:15:19,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T16:15:19,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-11T16:15:19,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.0840 sec 2024-11-11T16:15:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T16:15:19,696 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:19,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T16:15:19,699 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T16:15:19,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:19,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:19,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-11T16:15:19,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:19,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T16:15:19,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:19,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T16:15:19,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:19,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T16:15:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T16:15:19,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/56442c328b71408ab70d82f724759a54 is 50, key is test_row_0/A:col10/1731341719800/Put/seqid=0 2024-11-11T16:15:19,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341779806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341779808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341779813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341779815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341779817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742084_1260 (size=12301) 2024-11-11T16:15:19,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/56442c328b71408ab70d82f724759a54 2024-11-11T16:15:19,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/2cd6fed8634d4fffba691485f14bff84 is 50, key is test_row_0/B:col10/1731341719800/Put/seqid=0 2024-11-11T16:15:19,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T16:15:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:19,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:19,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742085_1261 (size=12301) 2024-11-11T16:15:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341779926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341779926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341779926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341779927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T16:15:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T16:15:20,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341780127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341780135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341780136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341780136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341780137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T16:15:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/2cd6fed8634d4fffba691485f14bff84 2024-11-11T16:15:20,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/a4ac68b29a8f48b296394bbe50a7e39b is 50, key is test_row_0/C:col10/1731341719800/Put/seqid=0 2024-11-11T16:15:20,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742086_1262 (size=12301) 2024-11-11T16:15:20,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T16:15:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T16:15:20,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341780446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341780446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341780446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341780456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T16:15:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T16:15:20,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:20,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:20,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341780645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:20,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/a4ac68b29a8f48b296394bbe50a7e39b 2024-11-11T16:15:20,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/56442c328b71408ab70d82f724759a54 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/56442c328b71408ab70d82f724759a54 2024-11-11T16:15:20,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/56442c328b71408ab70d82f724759a54, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T16:15:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/2cd6fed8634d4fffba691485f14bff84 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/2cd6fed8634d4fffba691485f14bff84 2024-11-11T16:15:20,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/2cd6fed8634d4fffba691485f14bff84, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T16:15:20,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/a4ac68b29a8f48b296394bbe50a7e39b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/a4ac68b29a8f48b296394bbe50a7e39b 2024-11-11T16:15:20,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/a4ac68b29a8f48b296394bbe50a7e39b, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T16:15:20,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=33.54 KB/34350 for 8a30e5caa9f1f33bab5e05117c34d967 in 915ms, sequenceid=382, compaction requested=true 2024-11-11T16:15:20,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:20,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:20,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:20,715 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:20,716 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:20,717 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:20,717 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:20,717 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:20,717 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,717 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:20,717 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/42ff5a7e4b8d463bb6bce1ce4842d5ad, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/7b3bc2ff42384ad0bf6acdf3e55d9679, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/56442c328b71408ab70d82f724759a54] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.2 K 2024-11-11T16:15:20,717 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,717 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0db7166dc35548039e4b7e77a48f0c90, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d93ce57d9ab847aabcdbf16297eb577a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/2cd6fed8634d4fffba691485f14bff84] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.2 K 2024-11-11T16:15:20,718 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42ff5a7e4b8d463bb6bce1ce4842d5ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731341718699 2024-11-11T16:15:20,718 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db7166dc35548039e4b7e77a48f0c90, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731341718699 2024-11-11T16:15:20,718 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d93ce57d9ab847aabcdbf16297eb577a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341718720 2024-11-11T16:15:20,718 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b3bc2ff42384ad0bf6acdf3e55d9679, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341718720 2024-11-11T16:15:20,719 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56442c328b71408ab70d82f724759a54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731341719460 2024-11-11T16:15:20,719 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd6fed8634d4fffba691485f14bff84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731341719460 2024-11-11T16:15:20,729 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:20,730 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/5ba08f8774904763b73c372296da1c65 is 50, key is test_row_0/B:col10/1731341719800/Put/seqid=0 2024-11-11T16:15:20,751 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:20,752 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/3c359511f11447a5abe731e2bd4d9245 is 50, key is test_row_0/A:col10/1731341719800/Put/seqid=0 2024-11-11T16:15:20,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T16:15:20,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,785 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T16:15:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:20,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:20,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742087_1263 (size=12609) 2024-11-11T16:15:20,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742088_1264 (size=12609) 2024-11-11T16:15:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d8f0c57479584eb4898e2df38157b926 is 50, key is test_row_0/A:col10/1731341719802/Put/seqid=0 2024-11-11T16:15:20,816 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/5ba08f8774904763b73c372296da1c65 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/5ba08f8774904763b73c372296da1c65 2024-11-11T16:15:20,825 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/3c359511f11447a5abe731e2bd4d9245 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3c359511f11447a5abe731e2bd4d9245 2024-11-11T16:15:20,829 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into 5ba08f8774904763b73c372296da1c65(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:20,829 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:20,829 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341720715; duration=0sec 2024-11-11T16:15:20,829 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:20,829 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:20,830 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:20,833 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:20,833 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:20,834 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:20,834 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/bcde75f8dc8c4331a06737b7c818c986, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/0607b13cc08741e48c3547457ddcdec7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/a4ac68b29a8f48b296394bbe50a7e39b] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.2 K 2024-11-11T16:15:20,834 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into 3c359511f11447a5abe731e2bd4d9245(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:20,834 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:20,834 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341720715; duration=0sec 2024-11-11T16:15:20,834 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:20,834 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:20,834 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting bcde75f8dc8c4331a06737b7c818c986, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731341718699 2024-11-11T16:15:20,835 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0607b13cc08741e48c3547457ddcdec7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731341718720 2024-11-11T16:15:20,835 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a4ac68b29a8f48b296394bbe50a7e39b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731341719460 2024-11-11T16:15:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T16:15:20,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742089_1265 (size=9857) 2024-11-11T16:15:20,867 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:20,867 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/4c1346835a1c4a25ae0984f5b1e8e7fc is 50, key is test_row_0/C:col10/1731341719800/Put/seqid=0 2024-11-11T16:15:20,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742090_1266 (size=12609) 2024-11-11T16:15:20,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341720990, max timestamp=1731341721001Num uniques 859; 2024-11-11T16:15:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:21,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341720990, max timestamp=1731341721001Num uniques 859; 2024-11-11T16:15:21,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:21,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341720990, max timestamp=1731341721001Num uniques 859; 2024-11-11T16:15:21,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:21,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731341721026, max timestamp=1731341721046Num uniques 1018; 2024-11-11T16:15:21,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T16:15:21,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341721040, max timestamp=1731341721046Num uniques 1018; 2024-11-11T16:15:21,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T16:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7686, totalHeapSize=17712, min timestamp=1731341721040, max timestamp=1731341721050Num uniques 1018; 2024-11-11T16:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46616, new segment data size=7686, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T16:15:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341721113, max timestamp=1731341721123Num uniques 1127; 2024-11-11T16:15:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-11T16:15:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341721113, max timestamp=1731341721123Num uniques 1027; 2024-11-11T16:15:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341721113, max timestamp=1731341721123Num uniques 1027; 2024-11-11T16:15:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T16:15:21,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T16:15:21,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731341721131, max timestamp=1731341721164Num uniques 1036; 2024-11-11T16:15:21,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731341721131, max timestamp=1731341721164Num uniques 1036; 2024-11-11T16:15:21,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731341721131, max timestamp=1731341721164Num uniques 1036; 2024-11-11T16:15:21,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-11T16:15:21,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-11T16:15:21,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-11T16:15:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341781227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341781228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341781229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341781237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,265 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d8f0c57479584eb4898e2df38157b926 2024-11-11T16:15:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/d367402bae524481a50f8f4f2e71c641 is 50, key is test_row_0/B:col10/1731341719802/Put/seqid=0 2024-11-11T16:15:21,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742091_1267 (size=9857) 2024-11-11T16:15:21,335 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/d367402bae524481a50f8f4f2e71c641 2024-11-11T16:15:21,337 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/4c1346835a1c4a25ae0984f5b1e8e7fc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c1346835a1c4a25ae0984f5b1e8e7fc 2024-11-11T16:15:21,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341781346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341781347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341781347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,349 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into 4c1346835a1c4a25ae0984f5b1e8e7fc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,350 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:21,350 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341720716; duration=0sec 2024-11-11T16:15:21,350 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:21,350 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/8c8ea86d38b7440281610d9011b487fe is 50, key is test_row_0/C:col10/1731341719802/Put/seqid=0 2024-11-11T16:15:21,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341781357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742092_1268 (size=9857) 2024-11-11T16:15:21,407 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/8c8ea86d38b7440281610d9011b487fe 2024-11-11T16:15:21,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d8f0c57479584eb4898e2df38157b926 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8f0c57479584eb4898e2df38157b926 2024-11-11T16:15:21,433 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8f0c57479584eb4898e2df38157b926, entries=100, sequenceid=390, filesize=9.6 K 2024-11-11T16:15:21,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/d367402bae524481a50f8f4f2e71c641 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d367402bae524481a50f8f4f2e71c641 2024-11-11T16:15:21,441 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d367402bae524481a50f8f4f2e71c641, entries=100, sequenceid=390, filesize=9.6 K 2024-11-11T16:15:21,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/8c8ea86d38b7440281610d9011b487fe as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/8c8ea86d38b7440281610d9011b487fe 2024-11-11T16:15:21,446 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/8c8ea86d38b7440281610d9011b487fe, entries=100, sequenceid=390, filesize=9.6 K 2024-11-11T16:15:21,447 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 8a30e5caa9f1f33bab5e05117c34d967 in 663ms, sequenceid=390, compaction requested=false 2024-11-11T16:15:21,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T16:15:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T16:15:21,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T16:15:21,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7480 sec 2024-11-11T16:15:21,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.7560 sec 2024-11-11T16:15:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:21,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341721190, max timestamp=1731341721558Num uniques 1195; 2024-11-11T16:15:21,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-11T16:15:21,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341721190, max timestamp=1731341721558Num uniques 1195; 2024-11-11T16:15:21,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-11T16:15:21,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=116.29 KB heapSize=320.14 KB 2024-11-11T16:15:21,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341721190, max timestamp=1731341721558Num uniques 1195; 2024-11-11T16:15:21,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-11T16:15:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:21,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/6ed8f41df0104812af894b3842eb9399 is 50, key is test_row_0/A:col10/1731341721561/Put/seqid=0 2024-11-11T16:15:21,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742093_1269 (size=12301) 2024-11-11T16:15:21,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341721595, max timestamp=1731341721607Num uniques 859; 2024-11-11T16:15:21,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:21,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341721595, max timestamp=1731341721607Num uniques 859; 2024-11-11T16:15:21,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341721595, max timestamp=1731341721607Num uniques 859; 2024-11-11T16:15:21,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:21,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:21,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731341721642, max timestamp=1731341721655Num uniques 1018; 2024-11-11T16:15:21,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T16:15:21,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731341721642, max timestamp=1731341721655Num uniques 1018; 2024-11-11T16:15:21,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T16:15:21,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731341721642, max timestamp=1731341721655Num uniques 1018; 2024-11-11T16:15:21,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T16:15:21,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341721689, max timestamp=1731341721709Num uniques 1127; 2024-11-11T16:15:21,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-11T16:15:21,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341721689, max timestamp=1731341721709Num uniques 1127; 2024-11-11T16:15:21,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-11T16:15:21,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341721689, max timestamp=1731341721709Num uniques 1127; 2024-11-11T16:15:21,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-11T16:15:21,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341721754, max timestamp=1731341721782Num uniques 1086; 2024-11-11T16:15:21,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-11T16:15:21,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341721754, max timestamp=1731341721782Num uniques 1086; 2024-11-11T16:15:21,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-11T16:15:21,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341721754, max timestamp=1731341721782Num uniques 1086; 2024-11-11T16:15:21,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-11T16:15:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T16:15:21,851 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:21,852 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:21,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341721830, max timestamp=1731341721855Num uniques 1045; 2024-11-11T16:15:21,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T16:15:21,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341721830, max timestamp=1731341721855Num uniques 1045; 2024-11-11T16:15:21,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T16:15:21,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341721830, max timestamp=1731341721855Num uniques 1045; 2024-11-11T16:15:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T16:15:21,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T16:15:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T16:15:21,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:21,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:21,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341781912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341781914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341781914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341781916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341781916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T16:15:22,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/6ed8f41df0104812af894b3842eb9399 2024-11-11T16:15:22,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/a55782e168d147f5b836194ec64220e7 is 50, key is test_row_0/B:col10/1731341721561/Put/seqid=0 2024-11-11T16:15:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341782026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341782032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:22,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341782036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341782036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341782036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742094_1270 (size=12301) 2024-11-11T16:15:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T16:15:22,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341782235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341782246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341782246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341782247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341782247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:22,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:22,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/a55782e168d147f5b836194ec64220e7 2024-11-11T16:15:22,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/da5d6e88a9f044f1a62b57e31b35bf4d is 50, key is test_row_0/C:col10/1731341721561/Put/seqid=0 2024-11-11T16:15:22,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742095_1271 (size=12301) 2024-11-11T16:15:22,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/da5d6e88a9f044f1a62b57e31b35bf4d 2024-11-11T16:15:22,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/6ed8f41df0104812af894b3842eb9399 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/6ed8f41df0104812af894b3842eb9399 2024-11-11T16:15:22,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/6ed8f41df0104812af894b3842eb9399, entries=150, sequenceid=484, filesize=12.0 K 2024-11-11T16:15:22,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/a55782e168d147f5b836194ec64220e7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/a55782e168d147f5b836194ec64220e7 2024-11-11T16:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T16:15:22,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/a55782e168d147f5b836194ec64220e7, entries=150, sequenceid=484, filesize=12.0 K 2024-11-11T16:15:22,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:22,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/da5d6e88a9f044f1a62b57e31b35bf4d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/da5d6e88a9f044f1a62b57e31b35bf4d 2024-11-11T16:15:22,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/da5d6e88a9f044f1a62b57e31b35bf4d, entries=150, sequenceid=484, filesize=12.0 K 2024-11-11T16:15:22,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~102.66 KB/105120, currentSize=147.60 KB/151140 for 8a30e5caa9f1f33bab5e05117c34d967 in 959ms, sequenceid=484, compaction requested=true 2024-11-11T16:15:22,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:22,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:22,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:22,521 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:22,521 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:22,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:22,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:22,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:22,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:22,523 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:22,523 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:22,523 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,523 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3c359511f11447a5abe731e2bd4d9245, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8f0c57479584eb4898e2df38157b926, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/6ed8f41df0104812af894b3842eb9399] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=34.0 K 2024-11-11T16:15:22,524 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:22,524 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:22,524 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,524 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c359511f11447a5abe731e2bd4d9245, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731341719460 2024-11-11T16:15:22,524 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/5ba08f8774904763b73c372296da1c65, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d367402bae524481a50f8f4f2e71c641, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/a55782e168d147f5b836194ec64220e7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=34.0 K 2024-11-11T16:15:22,524 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f0c57479584eb4898e2df38157b926, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731341719802 2024-11-11T16:15:22,525 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ed8f41df0104812af894b3842eb9399, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731341721190 2024-11-11T16:15:22,525 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba08f8774904763b73c372296da1c65, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731341719460 2024-11-11T16:15:22,526 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d367402bae524481a50f8f4f2e71c641, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731341719802 2024-11-11T16:15:22,527 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a55782e168d147f5b836194ec64220e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731341721190 2024-11-11T16:15:22,546 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:22,547 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/bdaab3b0f64f4605982b69b1422fdf36 is 50, key is test_row_0/A:col10/1731341721561/Put/seqid=0 2024-11-11T16:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:22,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341721901, max timestamp=1731341722552Num uniques 1104; 2024-11-11T16:15:22,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341721901, max timestamp=1731341722552Num uniques 1104; 2024-11-11T16:15:22,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-11T16:15:22,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-11T16:15:22,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341721901, max timestamp=1731341722552Num uniques 1104; 2024-11-11T16:15:22,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-11T16:15:22,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=26.84 KB heapSize=66.70 KB 2024-11-11T16:15:22,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:22,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:22,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:22,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:22,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:22,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:22,565 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:22,566 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/23b462a92c3f4c449fb9f138104d184b is 50, key is test_row_0/B:col10/1731341721561/Put/seqid=0 2024-11-11T16:15:22,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742096_1272 (size=12711) 2024-11-11T16:15:22,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/13ec71476ec649e49a73061448e1beab is 50, key is test_row_0/A:col10/1731341722563/Put/seqid=0 2024-11-11T16:15:22,585 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/bdaab3b0f64f4605982b69b1422fdf36 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/bdaab3b0f64f4605982b69b1422fdf36 2024-11-11T16:15:22,592 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into bdaab3b0f64f4605982b69b1422fdf36(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:22,592 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:22,592 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341722521; duration=0sec 2024-11-11T16:15:22,592 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:22,592 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:22,592 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:22,593 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:22,593 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:22,593 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,594 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c1346835a1c4a25ae0984f5b1e8e7fc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/8c8ea86d38b7440281610d9011b487fe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/da5d6e88a9f044f1a62b57e31b35bf4d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=34.0 K 2024-11-11T16:15:22,594 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c1346835a1c4a25ae0984f5b1e8e7fc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731341719460 2024-11-11T16:15:22,595 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8ea86d38b7440281610d9011b487fe, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731341719802 2024-11-11T16:15:22,595 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5d6e88a9f044f1a62b57e31b35bf4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731341721190 2024-11-11T16:15:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742097_1273 (size=12711) 2024-11-11T16:15:22,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742098_1274 (size=12301) 2024-11-11T16:15:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341722602, max timestamp=1731341722610Num uniques 859; 2024-11-11T16:15:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341722602, max timestamp=1731341722611Num uniques 859; 2024-11-11T16:15:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341722602, max timestamp=1731341722610Num uniques 859; 2024-11-11T16:15:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:22,622 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:22,622 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/4c42d6c6d3254642a45b7fb508b5f016 is 50, key is test_row_0/C:col10/1731341721561/Put/seqid=0 2024-11-11T16:15:22,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742099_1275 (size=12711) 2024-11-11T16:15:22,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731341722671, max timestamp=1731341722675Num uniques 1018; 2024-11-11T16:15:22,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T16:15:22,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731341722671, max timestamp=1731341722675Num uniques 1018; 2024-11-11T16:15:22,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T16:15:22,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731341722671, max timestamp=1731341722675Num uniques 1018; 2024-11-11T16:15:22,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T16:15:22,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341722708, max timestamp=1731341722723Num uniques 1077; 2024-11-11T16:15:22,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-11T16:15:22,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341722708, max timestamp=1731341722723Num uniques 1077; 2024-11-11T16:15:22,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-11T16:15:22,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341722708, max timestamp=1731341722723Num uniques 1077; 2024-11-11T16:15:22,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-11T16:15:22,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:22,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341722797, max timestamp=1731341722836Num uniques 1036; 2024-11-11T16:15:22,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341722797, max timestamp=1731341722841Num uniques 1036; 2024-11-11T16:15:22,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731341722797, max timestamp=1731341722836Num uniques 1036; 2024-11-11T16:15:22,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T16:15:22,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T16:15:22,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T16:15:22,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731341722854, max timestamp=1731341722920Num uniques 1045; 2024-11-11T16:15:22,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=18090, suffix heap size=136648,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=395 2024-11-11T16:15:22,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341722854, max timestamp=1731341722920Num uniques 1045; 2024-11-11T16:15:22,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T16:15:22,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341722854, max timestamp=1731341722920Num uniques 1045; 2024-11-11T16:15:22,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T16:15:22,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:22,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T16:15:23,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/13ec71476ec649e49a73061448e1beab 2024-11-11T16:15:23,019 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/23b462a92c3f4c449fb9f138104d184b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/23b462a92c3f4c449fb9f138104d184b 2024-11-11T16:15:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341782995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341782994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341783020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341783021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341783025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4eeedaf233fa446e8134986b8d824817 is 50, key is test_row_0/B:col10/1731341722563/Put/seqid=0 2024-11-11T16:15:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,037 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into 23b462a92c3f4c449fb9f138104d184b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:23,037 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,037 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341722521; duration=0sec 2024-11-11T16:15:23,037 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:23,037 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:23,045 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/4c42d6c6d3254642a45b7fb508b5f016 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c42d6c6d3254642a45b7fb508b5f016 2024-11-11T16:15:23,057 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into 4c42d6c6d3254642a45b7fb508b5f016(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:23,057 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,057 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341722521; duration=0sec 2024-11-11T16:15:23,057 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:23,057 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:23,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742100_1276 (size=12301) 2024-11-11T16:15:23,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4eeedaf233fa446e8134986b8d824817 2024-11-11T16:15:23,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/33c214ed2bb84e7da4548dc018736332 is 50, key is test_row_0/C:col10/1731341722563/Put/seqid=0 2024-11-11T16:15:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341783126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:23,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:23,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:23,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:23,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:23,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:23,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341783136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341783136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742101_1277 (size=12301) 2024-11-11T16:15:23,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/33c214ed2bb84e7da4548dc018736332 2024-11-11T16:15:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341783146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341783146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/13ec71476ec649e49a73061448e1beab as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/13ec71476ec649e49a73061448e1beab 2024-11-11T16:15:23,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/13ec71476ec649e49a73061448e1beab, entries=150, sequenceid=592, filesize=12.0 K 2024-11-11T16:15:23,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4eeedaf233fa446e8134986b8d824817 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4eeedaf233fa446e8134986b8d824817 2024-11-11T16:15:23,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4eeedaf233fa446e8134986b8d824817, entries=150, sequenceid=592, filesize=12.0 K 2024-11-11T16:15:23,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/33c214ed2bb84e7da4548dc018736332 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/33c214ed2bb84e7da4548dc018736332 2024-11-11T16:15:23,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/33c214ed2bb84e7da4548dc018736332, entries=150, sequenceid=592, filesize=12.0 K 2024-11-11T16:15:23,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=154.31 KB/158010 for 8a30e5caa9f1f33bab5e05117c34d967 in 613ms, sequenceid=592, compaction requested=false 2024-11-11T16:15:23,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T16:15:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:23,286 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-11T16:15:23,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:23,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:23,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:23,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:23,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:23,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/af712e9f23924e43a58fc6a89936e4e4 is 50, key is test_row_0/A:col10/1731341722992/Put/seqid=0 2024-11-11T16:15:23,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742102_1278 (size=12301) 2024-11-11T16:15:23,317 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/af712e9f23924e43a58fc6a89936e4e4 2024-11-11T16:15:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/65f3ee35c8364e17baaaf6f857b78066 is 50, key is test_row_0/B:col10/1731341722992/Put/seqid=0 2024-11-11T16:15:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:23,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:23,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742103_1279 (size=12301) 2024-11-11T16:15:23,345 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/65f3ee35c8364e17baaaf6f857b78066 2024-11-11T16:15:23,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2 is 50, key is test_row_0/C:col10/1731341722992/Put/seqid=0 2024-11-11T16:15:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341783354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341783356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341783357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341783357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341783356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742104_1280 (size=12301) 2024-11-11T16:15:23,394 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2 2024-11-11T16:15:23,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/af712e9f23924e43a58fc6a89936e4e4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/af712e9f23924e43a58fc6a89936e4e4 2024-11-11T16:15:23,409 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/af712e9f23924e43a58fc6a89936e4e4, entries=150, sequenceid=701, filesize=12.0 K 2024-11-11T16:15:23,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/65f3ee35c8364e17baaaf6f857b78066 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/65f3ee35c8364e17baaaf6f857b78066 2024-11-11T16:15:23,415 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/65f3ee35c8364e17baaaf6f857b78066, entries=150, sequenceid=701, filesize=12.0 K 2024-11-11T16:15:23,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2 2024-11-11T16:15:23,423 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2, entries=150, sequenceid=701, filesize=12.0 K 2024-11-11T16:15:23,425 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=33.54 KB/34350 for 8a30e5caa9f1f33bab5e05117c34d967 in 139ms, sequenceid=701, compaction requested=true 2024-11-11T16:15:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T16:15:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T16:15:23,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T16:15:23,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5450 sec 2024-11-11T16:15:23,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5770 sec 2024-11-11T16:15:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:23,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:23,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:23,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:23,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:23,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:23,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:23,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:23,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c8cfa0ac4bcb428eb37062c78df3fe10 is 50, key is test_row_0/A:col10/1731341723353/Put/seqid=0 2024-11-11T16:15:23,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742105_1281 (size=14741) 2024-11-11T16:15:23,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c8cfa0ac4bcb428eb37062c78df3fe10 2024-11-11T16:15:23,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/7c14332c3619492588532817ba9e64f8 is 50, key is test_row_0/B:col10/1731341723353/Put/seqid=0 2024-11-11T16:15:23,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742106_1282 (size=12301) 2024-11-11T16:15:23,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/7c14332c3619492588532817ba9e64f8 2024-11-11T16:15:23,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341723511, max timestamp=1731341723523Num uniques 859; 2024-11-11T16:15:23,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:23,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341723511, max timestamp=1731341723523Num uniques 859; 2024-11-11T16:15:23,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:23,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341723511, max timestamp=1731341723523Num uniques 859; 2024-11-11T16:15:23,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:23,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e955fa36654e413586dacb2788fc8e0d is 50, key is test_row_0/C:col10/1731341723353/Put/seqid=0 2024-11-11T16:15:23,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742107_1283 (size=12301) 2024-11-11T16:15:23,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e955fa36654e413586dacb2788fc8e0d 2024-11-11T16:15:23,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/c8cfa0ac4bcb428eb37062c78df3fe10 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c8cfa0ac4bcb428eb37062c78df3fe10 2024-11-11T16:15:23,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c8cfa0ac4bcb428eb37062c78df3fe10, entries=200, sequenceid=711, filesize=14.4 K 2024-11-11T16:15:23,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/7c14332c3619492588532817ba9e64f8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/7c14332c3619492588532817ba9e64f8 2024-11-11T16:15:23,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/7c14332c3619492588532817ba9e64f8, entries=150, sequenceid=711, filesize=12.0 K 2024-11-11T16:15:23,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e955fa36654e413586dacb2788fc8e0d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e955fa36654e413586dacb2788fc8e0d 2024-11-11T16:15:23,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e955fa36654e413586dacb2788fc8e0d, entries=150, sequenceid=711, filesize=12.0 K 2024-11-11T16:15:23,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=93.93 KB/96180 for 8a30e5caa9f1f33bab5e05117c34d967 in 97ms, sequenceid=711, compaction requested=true 2024-11-11T16:15:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,571 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:23,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:23,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:23,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:23,572 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:23,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:23,575 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:23,575 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:23,575 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:23,575 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:23,575 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:23,575 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/bdaab3b0f64f4605982b69b1422fdf36, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/13ec71476ec649e49a73061448e1beab, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/af712e9f23924e43a58fc6a89936e4e4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c8cfa0ac4bcb428eb37062c78df3fe10] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=50.8 K 2024-11-11T16:15:23,575 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:23,575 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/23b462a92c3f4c449fb9f138104d184b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4eeedaf233fa446e8134986b8d824817, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/65f3ee35c8364e17baaaf6f857b78066, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/7c14332c3619492588532817ba9e64f8] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=48.5 K 2024-11-11T16:15:23,576 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdaab3b0f64f4605982b69b1422fdf36, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731341721190 2024-11-11T16:15:23,576 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b462a92c3f4c449fb9f138104d184b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731341721190 2024-11-11T16:15:23,576 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13ec71476ec649e49a73061448e1beab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731341721911 2024-11-11T16:15:23,576 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eeedaf233fa446e8134986b8d824817, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731341721911 2024-11-11T16:15:23,577 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f3ee35c8364e17baaaf6f857b78066, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=701, earliestPutTs=1731341722992 2024-11-11T16:15:23,577 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting af712e9f23924e43a58fc6a89936e4e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=701, earliestPutTs=1731341722992 2024-11-11T16:15:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:23,578 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8cfa0ac4bcb428eb37062c78df3fe10, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731341723353 2024-11-11T16:15:23,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=107.34 KB heapSize=300.61 KB 2024-11-11T16:15:23,578 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c14332c3619492588532817ba9e64f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731341723353 2024-11-11T16:15:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:23,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/31e4b60fe4db44039b1aed5719c257bc is 50, key is test_row_0/A:col10/1731341723573/Put/seqid=0 2024-11-11T16:15:23,603 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:23,603 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e5a82f3054294ce9bbca69e155e75109 is 50, key is test_row_0/B:col10/1731341723353/Put/seqid=0 2024-11-11T16:15:23,604 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:23,605 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/34486384813943b8a256d2ccc4318110 is 50, key is test_row_0/A:col10/1731341723353/Put/seqid=0 2024-11-11T16:15:23,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742108_1284 (size=14741) 2024-11-11T16:15:23,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/31e4b60fe4db44039b1aed5719c257bc 2024-11-11T16:15:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341783609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/f0b70d97eae048f181bb86e0dd8f6265 is 50, key is test_row_0/B:col10/1731341723573/Put/seqid=0 2024-11-11T16:15:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742109_1285 (size=12847) 2024-11-11T16:15:23,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341783618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742110_1286 (size=12847) 2024-11-11T16:15:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341783621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742111_1287 (size=12301) 2024-11-11T16:15:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/f0b70d97eae048f181bb86e0dd8f6265 2024-11-11T16:15:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,636 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e5a82f3054294ce9bbca69e155e75109 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e5a82f3054294ce9bbca69e155e75109 2024-11-11T16:15:23,644 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/34486384813943b8a256d2ccc4318110 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/34486384813943b8a256d2ccc4318110 2024-11-11T16:15:23,649 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into e5a82f3054294ce9bbca69e155e75109(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:23,649 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,649 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=12, startTime=1731341723572; duration=0sec 2024-11-11T16:15:23,649 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:23,649 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:23,649 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:23,651 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into 34486384813943b8a256d2ccc4318110(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:23,651 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,651 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=12, startTime=1731341723571; duration=0sec 2024-11-11T16:15:23,651 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:23,651 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:23,652 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:23,652 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:23,652 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:23,652 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c42d6c6d3254642a45b7fb508b5f016, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/33c214ed2bb84e7da4548dc018736332, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e955fa36654e413586dacb2788fc8e0d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=48.5 K 2024-11-11T16:15:23,652 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c42d6c6d3254642a45b7fb508b5f016, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731341721190 2024-11-11T16:15:23,652 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c214ed2bb84e7da4548dc018736332, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731341721911 2024-11-11T16:15:23,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/b4a62678a6f1451cbbef41e56e0c430f is 50, key is test_row_0/C:col10/1731341723573/Put/seqid=0 2024-11-11T16:15:23,655 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 53cc78f6e5784a0fb2bc1b3b1a6c6bb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=701, earliestPutTs=1731341722992 2024-11-11T16:15:23,655 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e955fa36654e413586dacb2788fc8e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731341723353 2024-11-11T16:15:23,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742112_1288 (size=12301) 2024-11-11T16:15:23,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341783666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/b4a62678a6f1451cbbef41e56e0c430f 2024-11-11T16:15:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341783678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/31e4b60fe4db44039b1aed5719c257bc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/31e4b60fe4db44039b1aed5719c257bc 2024-11-11T16:15:23,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/31e4b60fe4db44039b1aed5719c257bc, entries=200, sequenceid=744, filesize=14.4 K 2024-11-11T16:15:23,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/f0b70d97eae048f181bb86e0dd8f6265 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/f0b70d97eae048f181bb86e0dd8f6265 2024-11-11T16:15:23,692 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:23,692 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e75c642c7a714e62875e65db5ac0dd60 is 50, key is test_row_0/C:col10/1731341723353/Put/seqid=0 2024-11-11T16:15:23,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/f0b70d97eae048f181bb86e0dd8f6265, entries=150, sequenceid=744, filesize=12.0 K 2024-11-11T16:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/b4a62678a6f1451cbbef41e56e0c430f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b4a62678a6f1451cbbef41e56e0c430f 2024-11-11T16:15:23,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b4a62678a6f1451cbbef41e56e0c430f, entries=150, sequenceid=744, filesize=12.0 K 2024-11-11T16:15:23,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~320.48 KB/328176, currentSize=73.80 KB/75570 for 8a30e5caa9f1f33bab5e05117c34d967 in 128ms, sequenceid=744, compaction requested=false 2024-11-11T16:15:23,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742113_1289 (size=12847) 2024-11-11T16:15:23,726 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e75c642c7a714e62875e65db5ac0dd60 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e75c642c7a714e62875e65db5ac0dd60 2024-11-11T16:15:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:23,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:15:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:23,734 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into e75c642c7a714e62875e65db5ac0dd60(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:23,734 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:23,734 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=12, startTime=1731341723572; duration=0sec 2024-11-11T16:15:23,734 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:23,734 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:23,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/b2c0edd9dddc47ab8e16f8b0033f32fa is 50, key is test_row_0/A:col10/1731341723616/Put/seqid=0 2024-11-11T16:15:23,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742114_1290 (size=12301) 2024-11-11T16:15:23,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/b2c0edd9dddc47ab8e16f8b0033f32fa 2024-11-11T16:15:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341783768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341783769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e002be85d1ba44e584c81265236631da is 50, key is test_row_0/B:col10/1731341723616/Put/seqid=0 2024-11-11T16:15:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341783775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742115_1291 (size=12301) 2024-11-11T16:15:23,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341783886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341783886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:23,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341783886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:23,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:23,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T16:15:24,016 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:24,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T16:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T16:15:24,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:24,021 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:24,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341784096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341784096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341784096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T16:15:24,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T16:15:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341784176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341784185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e002be85d1ba44e584c81265236631da 2024-11-11T16:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/b57406d0c6d94f5e963bb399fac4a517 is 50, key is test_row_0/C:col10/1731341723616/Put/seqid=0 2024-11-11T16:15:24,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742116_1292 (size=12301) 2024-11-11T16:15:24,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T16:15:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T16:15:24,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341784434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341784434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341784435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T16:15:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/b57406d0c6d94f5e963bb399fac4a517 2024-11-11T16:15:24,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T16:15:24,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/b2c0edd9dddc47ab8e16f8b0033f32fa as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b2c0edd9dddc47ab8e16f8b0033f32fa 2024-11-11T16:15:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:24,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b2c0edd9dddc47ab8e16f8b0033f32fa, entries=150, sequenceid=761, filesize=12.0 K 2024-11-11T16:15:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T16:15:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e002be85d1ba44e584c81265236631da as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e002be85d1ba44e584c81265236631da 2024-11-11T16:15:24,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e002be85d1ba44e584c81265236631da, entries=150, sequenceid=761, filesize=12.0 K 2024-11-11T16:15:24,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/b57406d0c6d94f5e963bb399fac4a517 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b57406d0c6d94f5e963bb399fac4a517 2024-11-11T16:15:24,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b57406d0c6d94f5e963bb399fac4a517, entries=150, sequenceid=761, filesize=12.0 K 2024-11-11T16:15:24,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8a30e5caa9f1f33bab5e05117c34d967 in 930ms, sequenceid=761, compaction requested=true 2024-11-11T16:15:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:24,659 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:24,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:24,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:24,660 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:24,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:24,662 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:24,663 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:24,663 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,663 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e5a82f3054294ce9bbca69e155e75109, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/f0b70d97eae048f181bb86e0dd8f6265, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e002be85d1ba44e584c81265236631da] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.6 K 2024-11-11T16:15:24,663 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:24,663 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:24,663 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,663 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/34486384813943b8a256d2ccc4318110, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/31e4b60fe4db44039b1aed5719c257bc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b2c0edd9dddc47ab8e16f8b0033f32fa] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=39.0 K 2024-11-11T16:15:24,664 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a82f3054294ce9bbca69e155e75109, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731341723353 2024-11-11T16:15:24,664 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 34486384813943b8a256d2ccc4318110, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731341723353 2024-11-11T16:15:24,665 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0b70d97eae048f181bb86e0dd8f6265, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1731341723543 2024-11-11T16:15:24,665 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e4b60fe4db44039b1aed5719c257bc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1731341723543 2024-11-11T16:15:24,665 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c0edd9dddc47ab8e16f8b0033f32fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=761, earliestPutTs=1731341723607 2024-11-11T16:15:24,665 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e002be85d1ba44e584c81265236631da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=761, earliestPutTs=1731341723607 2024-11-11T16:15:24,679 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:24,680 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/ede9e510a706428c9869174bc39664f5 is 50, key is test_row_0/B:col10/1731341723616/Put/seqid=0 2024-11-11T16:15:24,684 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#250 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:24,684 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d8697d9cdc0c43b6998c6dba24ef7e5d is 50, key is test_row_0/A:col10/1731341723616/Put/seqid=0 2024-11-11T16:15:24,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742117_1293 (size=12949) 2024-11-11T16:15:24,755 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/ede9e510a706428c9869174bc39664f5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/ede9e510a706428c9869174bc39664f5 2024-11-11T16:15:24,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742118_1294 (size=12949) 2024-11-11T16:15:24,778 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into ede9e510a706428c9869174bc39664f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:24,778 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:24,778 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341724659; duration=0sec 2024-11-11T16:15:24,778 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:24,778 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:24,778 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:24,779 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:24,779 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:24,780 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,780 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e75c642c7a714e62875e65db5ac0dd60, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b4a62678a6f1451cbbef41e56e0c430f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b57406d0c6d94f5e963bb399fac4a517] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.6 K 2024-11-11T16:15:24,780 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e75c642c7a714e62875e65db5ac0dd60, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1731341723353 2024-11-11T16:15:24,781 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a62678a6f1451cbbef41e56e0c430f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1731341723543 2024-11-11T16:15:24,781 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57406d0c6d94f5e963bb399fac4a517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=761, earliestPutTs=1731341723607 2024-11-11T16:15:24,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T16:15:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:24,795 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T16:15:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:24,808 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:24,808 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/12a3dbdce5624a3aa2e3bec6a34d4141 is 50, key is test_row_0/C:col10/1731341723616/Put/seqid=0 2024-11-11T16:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/727f91880faa443a869271b730731527 is 50, key is test_row_0/A:col10/1731341723766/Put/seqid=0 2024-11-11T16:15:24,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742120_1296 (size=12301) 2024-11-11T16:15:24,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742119_1295 (size=12949) 2024-11-11T16:15:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:24,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:24,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341784992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:24,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341784992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341784997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341785106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341785106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341785106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T16:15:25,172 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d8697d9cdc0c43b6998c6dba24ef7e5d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8697d9cdc0c43b6998c6dba24ef7e5d 2024-11-11T16:15:25,181 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into d8697d9cdc0c43b6998c6dba24ef7e5d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:25,181 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:25,181 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341724659; duration=0sec 2024-11-11T16:15:25,181 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:25,181 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341785195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,199 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 8a30e5caa9f1f33bab5e05117c34d967 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341785205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,211 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 8a30e5caa9f1f33bab5e05117c34d967 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,257 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=780 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/727f91880faa443a869271b730731527 2024-11-11T16:15:25,263 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/12a3dbdce5624a3aa2e3bec6a34d4141 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/12a3dbdce5624a3aa2e3bec6a34d4141 2024-11-11T16:15:25,269 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into 12a3dbdce5624a3aa2e3bec6a34d4141(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:25,269 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:25,270 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341724660; duration=0sec 2024-11-11T16:15:25,270 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:25,270 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:25,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/59acb941b69b4dd186e8491605ec05cc is 50, key is test_row_0/B:col10/1731341723766/Put/seqid=0 2024-11-11T16:15:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742121_1297 (size=12301) 2024-11-11T16:15:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341785316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341785316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341785316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341785625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341785625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341785625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:25,685 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=780 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/59acb941b69b4dd186e8491605ec05cc 2024-11-11T16:15:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/14d5bd1979074a79b35aeb9f4a34aad0 is 50, key is test_row_0/C:col10/1731341723766/Put/seqid=0 2024-11-11T16:15:25,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742122_1298 (size=12301) 2024-11-11T16:15:25,730 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=780 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/14d5bd1979074a79b35aeb9f4a34aad0 2024-11-11T16:15:25,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/727f91880faa443a869271b730731527 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/727f91880faa443a869271b730731527 2024-11-11T16:15:25,743 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/727f91880faa443a869271b730731527, entries=150, sequenceid=780, filesize=12.0 K 2024-11-11T16:15:25,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/59acb941b69b4dd186e8491605ec05cc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/59acb941b69b4dd186e8491605ec05cc 2024-11-11T16:15:25,752 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/59acb941b69b4dd186e8491605ec05cc, entries=150, sequenceid=780, filesize=12.0 K 2024-11-11T16:15:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/14d5bd1979074a79b35aeb9f4a34aad0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/14d5bd1979074a79b35aeb9f4a34aad0 2024-11-11T16:15:25,757 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/14d5bd1979074a79b35aeb9f4a34aad0, entries=150, sequenceid=780, filesize=12.0 K 2024-11-11T16:15:25,758 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8a30e5caa9f1f33bab5e05117c34d967 in 963ms, sequenceid=780, compaction requested=false 2024-11-11T16:15:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T16:15:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T16:15:25,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T16:15:25,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7390 sec 2024-11-11T16:15:25,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7460 sec 2024-11-11T16:15:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:26,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T16:15:26,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:26,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:26,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:26,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:26,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:26,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:26,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/fad477392685431f82524dd62c414e59 is 50, key is test_row_0/A:col10/1731341724992/Put/seqid=0 2024-11-11T16:15:26,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742123_1299 (size=14741) 2024-11-11T16:15:26,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/fad477392685431f82524dd62c414e59 2024-11-11T16:15:26,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b9d7d8ebed35473c98f8ee42c885c885 is 50, key is test_row_0/B:col10/1731341724992/Put/seqid=0 2024-11-11T16:15:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T16:15:26,166 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:26,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T16:15:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T16:15:26,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341786165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,169 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:26,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341786166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341786168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742124_1300 (size=12301) 2024-11-11T16:15:26,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b9d7d8ebed35473c98f8ee42c885c885 2024-11-11T16:15:26,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/83cf328957a9471c9177f6f23a932c4e is 50, key is test_row_0/C:col10/1731341724992/Put/seqid=0 2024-11-11T16:15:26,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742125_1301 (size=12301) 2024-11-11T16:15:26,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=799 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/83cf328957a9471c9177f6f23a932c4e 2024-11-11T16:15:26,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/fad477392685431f82524dd62c414e59 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fad477392685431f82524dd62c414e59 2024-11-11T16:15:26,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fad477392685431f82524dd62c414e59, entries=200, sequenceid=799, filesize=14.4 K 2024-11-11T16:15:26,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b9d7d8ebed35473c98f8ee42c885c885 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b9d7d8ebed35473c98f8ee42c885c885 2024-11-11T16:15:26,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b9d7d8ebed35473c98f8ee42c885c885, entries=150, sequenceid=799, filesize=12.0 K 2024-11-11T16:15:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/83cf328957a9471c9177f6f23a932c4e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/83cf328957a9471c9177f6f23a932c4e 2024-11-11T16:15:26,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/83cf328957a9471c9177f6f23a932c4e, entries=150, sequenceid=799, filesize=12.0 K 2024-11-11T16:15:26,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8a30e5caa9f1f33bab5e05117c34d967 in 89ms, sequenceid=799, compaction requested=true 2024-11-11T16:15:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:26,229 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:26,229 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:26,229 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:26,230 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:26,230 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,230 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8697d9cdc0c43b6998c6dba24ef7e5d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/727f91880faa443a869271b730731527, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fad477392685431f82524dd62c414e59] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=39.1 K 2024-11-11T16:15:26,230 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:26,230 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:26,230 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,231 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/ede9e510a706428c9869174bc39664f5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/59acb941b69b4dd186e8491605ec05cc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b9d7d8ebed35473c98f8ee42c885c885] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.7 K 2024-11-11T16:15:26,231 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8697d9cdc0c43b6998c6dba24ef7e5d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=761, earliestPutTs=1731341723607 2024-11-11T16:15:26,231 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ede9e510a706428c9869174bc39664f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=761, earliestPutTs=1731341723607 2024-11-11T16:15:26,231 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 727f91880faa443a869271b730731527, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=780, earliestPutTs=1731341723766 2024-11-11T16:15:26,231 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 59acb941b69b4dd186e8491605ec05cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=780, earliestPutTs=1731341723766 2024-11-11T16:15:26,232 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting fad477392685431f82524dd62c414e59, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1731341724960 2024-11-11T16:15:26,233 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b9d7d8ebed35473c98f8ee42c885c885, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1731341724960 2024-11-11T16:15:26,242 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:26,243 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/841093c93b3b45fab2f78aa41750ea9c is 50, key is test_row_0/A:col10/1731341724992/Put/seqid=0 2024-11-11T16:15:26,246 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:26,248 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/02ac2a70f3d84efd9684be5b238e6856 is 50, key is test_row_0/B:col10/1731341724992/Put/seqid=0 2024-11-11T16:15:26,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742127_1303 (size=13051) 2024-11-11T16:15:26,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742126_1302 (size=13051) 2024-11-11T16:15:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T16:15:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:26,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T16:15:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:26,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/b3c53bc6aac84e43a142c3b9be69f7e3 is 50, key is test_row_0/A:col10/1731341726164/Put/seqid=0 2024-11-11T16:15:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341786304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341786303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742128_1304 (size=12301) 2024-11-11T16:15:26,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341786310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341786416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341786416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341786425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:26,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:26,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T16:15:26,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341786626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341786626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341786635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,659 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/02ac2a70f3d84efd9684be5b238e6856 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/02ac2a70f3d84efd9684be5b238e6856 2024-11-11T16:15:26,664 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into 02ac2a70f3d84efd9684be5b238e6856(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:26,664 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:26,664 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341726229; duration=0sec 2024-11-11T16:15:26,665 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:26,665 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:26,665 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:26,665 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:26,666 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:26,666 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,666 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/12a3dbdce5624a3aa2e3bec6a34d4141, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/14d5bd1979074a79b35aeb9f4a34aad0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/83cf328957a9471c9177f6f23a932c4e] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.7 K 2024-11-11T16:15:26,666 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 12a3dbdce5624a3aa2e3bec6a34d4141, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=761, earliestPutTs=1731341723607 2024-11-11T16:15:26,666 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d5bd1979074a79b35aeb9f4a34aad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=780, earliestPutTs=1731341723766 2024-11-11T16:15:26,667 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 83cf328957a9471c9177f6f23a932c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1731341724960 2024-11-11T16:15:26,672 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/841093c93b3b45fab2f78aa41750ea9c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/841093c93b3b45fab2f78aa41750ea9c 2024-11-11T16:15:26,676 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:26,677 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/012397aed66e4583b2e583f4c09f0e68 is 50, key is test_row_0/C:col10/1731341724992/Put/seqid=0 2024-11-11T16:15:26,678 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into 841093c93b3b45fab2f78aa41750ea9c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:26,678 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:26,678 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341726228; duration=0sec 2024-11-11T16:15:26,678 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:26,679 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:26,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742129_1305 (size=13051) 2024-11-11T16:15:26,696 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/012397aed66e4583b2e583f4c09f0e68 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/012397aed66e4583b2e583f4c09f0e68 2024-11-11T16:15:26,703 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into 012397aed66e4583b2e583f4c09f0e68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:26,703 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:26,703 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341726229; duration=0sec 2024-11-11T16:15:26,703 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:26,703 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:26,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=817 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/b3c53bc6aac84e43a142c3b9be69f7e3 2024-11-11T16:15:26,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/24aed262011442648df9ac616706005b is 50, key is test_row_0/B:col10/1731341726164/Put/seqid=0 2024-11-11T16:15:26,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742130_1306 (size=12301) 2024-11-11T16:15:26,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T16:15:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341786936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341786936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:26,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:26,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:26,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341786945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:26,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:27,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=817 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/24aed262011442648df9ac616706005b 2024-11-11T16:15:27,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/17dadf7cd52a49be82c0692a81923f91 is 50, key is test_row_0/C:col10/1731341726164/Put/seqid=0 2024-11-11T16:15:27,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742131_1307 (size=12301) 2024-11-11T16:15:27,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341787216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:27,220 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 8a30e5caa9f1f33bab5e05117c34d967 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:27,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:27,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341787235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:27,239 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 8a30e5caa9f1f33bab5e05117c34d967 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:27,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T16:15:27,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:27,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341787445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341787445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:27,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341787456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:27,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:27,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=817 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/17dadf7cd52a49be82c0692a81923f91 2024-11-11T16:15:27,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/b3c53bc6aac84e43a142c3b9be69f7e3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b3c53bc6aac84e43a142c3b9be69f7e3 2024-11-11T16:15:27,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b3c53bc6aac84e43a142c3b9be69f7e3, entries=150, sequenceid=817, filesize=12.0 K 2024-11-11T16:15:27,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:27,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/24aed262011442648df9ac616706005b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/24aed262011442648df9ac616706005b 2024-11-11T16:15:27,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:27,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:27,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/24aed262011442648df9ac616706005b, entries=150, sequenceid=817, filesize=12.0 K 2024-11-11T16:15:27,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/17dadf7cd52a49be82c0692a81923f91 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/17dadf7cd52a49be82c0692a81923f91 2024-11-11T16:15:27,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/17dadf7cd52a49be82c0692a81923f91, entries=150, sequenceid=817, filesize=12.0 K 2024-11-11T16:15:27,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8a30e5caa9f1f33bab5e05117c34d967 in 1309ms, sequenceid=817, compaction requested=false 2024-11-11T16:15:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:27,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T16:15:27,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:27,725 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T16:15:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:27,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/f6fd47d7ec1c44749859187a77fff4d4 is 50, key is test_row_0/A:col10/1731341726302/Put/seqid=0 2024-11-11T16:15:27,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742132_1308 (size=12301) 2024-11-11T16:15:27,764 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/f6fd47d7ec1c44749859187a77fff4d4 2024-11-11T16:15:27,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/bc1bec641f4847798f9a3d9eedfc5b3b is 50, key is test_row_0/B:col10/1731341726302/Put/seqid=0 2024-11-11T16:15:27,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742133_1309 (size=12301) 2024-11-11T16:15:27,836 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/bc1bec641f4847798f9a3d9eedfc5b3b 2024-11-11T16:15:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/1ff522536f6a49098ece96720a643d52 is 50, key is test_row_0/C:col10/1731341726302/Put/seqid=0 2024-11-11T16:15:27,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742134_1310 (size=12301) 2024-11-11T16:15:28,259 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/1ff522536f6a49098ece96720a643d52 2024-11-11T16:15:28,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/f6fd47d7ec1c44749859187a77fff4d4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/f6fd47d7ec1c44749859187a77fff4d4 2024-11-11T16:15:28,273 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/f6fd47d7ec1c44749859187a77fff4d4, entries=150, sequenceid=834, filesize=12.0 K 2024-11-11T16:15:28,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/bc1bec641f4847798f9a3d9eedfc5b3b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/bc1bec641f4847798f9a3d9eedfc5b3b 2024-11-11T16:15:28,280 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/bc1bec641f4847798f9a3d9eedfc5b3b, entries=150, sequenceid=834, filesize=12.0 K 2024-11-11T16:15:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/1ff522536f6a49098ece96720a643d52 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/1ff522536f6a49098ece96720a643d52 2024-11-11T16:15:28,286 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/1ff522536f6a49098ece96720a643d52, entries=150, sequenceid=834, filesize=12.0 K 2024-11-11T16:15:28,287 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 8a30e5caa9f1f33bab5e05117c34d967 in 563ms, sequenceid=834, compaction requested=true 2024-11-11T16:15:28,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:28,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T16:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T16:15:28,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T16:15:28,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-11-11T16:15:28,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1280 sec 2024-11-11T16:15:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T16:15:28,319 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:28,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T16:15:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T16:15:28,321 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:28,322 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:28,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T16:15:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:28,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:28,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T16:15:28,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:28,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/3175689001ff425da8dda53e82a5e627 is 50, key is test_row_0/A:col10/1731341728471/Put/seqid=0 2024-11-11T16:15:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742135_1311 (size=12297) 2024-11-11T16:15:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341728493, max timestamp=1731341728508Num uniques 859; 2024-11-11T16:15:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341728501, max timestamp=1731341728508Num uniques 859; 2024-11-11T16:15:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341728501, max timestamp=1731341728508Num uniques 859; 2024-11-11T16:15:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:28,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341788541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341788545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341788546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T16:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T16:15:28,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341788655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341788655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341788656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T16:15:28,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:28,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341788865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341788865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341788866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:28,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/3175689001ff425da8dda53e82a5e627 2024-11-11T16:15:28,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/34311c7462184e508bc5c7de7d4d0553 is 50, key is test_row_0/B:col10/1731341728471/Put/seqid=0 2024-11-11T16:15:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742136_1312 (size=9857) 2024-11-11T16:15:28,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/34311c7462184e508bc5c7de7d4d0553 2024-11-11T16:15:28,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/5d90dce36a344289afefeb26a614b694 is 50, key is test_row_0/C:col10/1731341728471/Put/seqid=0 2024-11-11T16:15:28,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742137_1313 (size=9857) 2024-11-11T16:15:28,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T16:15:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:28,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T16:15:29,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T16:15:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:29,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:29,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341789175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341789176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341789176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T16:15:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:29,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/5d90dce36a344289afefeb26a614b694 2024-11-11T16:15:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/3175689001ff425da8dda53e82a5e627 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3175689001ff425da8dda53e82a5e627 2024-11-11T16:15:29,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3175689001ff425da8dda53e82a5e627, entries=150, sequenceid=844, filesize=12.0 K 2024-11-11T16:15:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/34311c7462184e508bc5c7de7d4d0553 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/34311c7462184e508bc5c7de7d4d0553 2024-11-11T16:15:29,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/34311c7462184e508bc5c7de7d4d0553, entries=100, sequenceid=844, filesize=9.6 K 2024-11-11T16:15:29,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/5d90dce36a344289afefeb26a614b694 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5d90dce36a344289afefeb26a614b694 2024-11-11T16:15:29,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5d90dce36a344289afefeb26a614b694, entries=100, sequenceid=844, filesize=9.6 K 2024-11-11T16:15:29,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 8a30e5caa9f1f33bab5e05117c34d967 in 893ms, sequenceid=844, compaction requested=true 2024-11-11T16:15:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:29,365 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:29,367 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49950 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:29,367 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:29,367 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,367 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/841093c93b3b45fab2f78aa41750ea9c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b3c53bc6aac84e43a142c3b9be69f7e3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/f6fd47d7ec1c44749859187a77fff4d4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3175689001ff425da8dda53e82a5e627] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=48.8 K 2024-11-11T16:15:29,368 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 841093c93b3b45fab2f78aa41750ea9c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1731341724960 2024-11-11T16:15:29,368 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3c53bc6aac84e43a142c3b9be69f7e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731341726164 2024-11-11T16:15:29,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:29,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:29,368 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6fd47d7ec1c44749859187a77fff4d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=834, earliestPutTs=1731341726292 2024-11-11T16:15:29,369 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3175689001ff425da8dda53e82a5e627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1731341728466 2024-11-11T16:15:29,370 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:29,372 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:29,372 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:29,372 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,372 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/02ac2a70f3d84efd9684be5b238e6856, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/24aed262011442648df9ac616706005b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/bc1bec641f4847798f9a3d9eedfc5b3b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/34311c7462184e508bc5c7de7d4d0553] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=46.4 K 2024-11-11T16:15:29,373 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ac2a70f3d84efd9684be5b238e6856, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1731341724960 2024-11-11T16:15:29,374 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 24aed262011442648df9ac616706005b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731341726164 2024-11-11T16:15:29,375 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1bec641f4847798f9a3d9eedfc5b3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=834, earliestPutTs=1731341726292 2024-11-11T16:15:29,375 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 34311c7462184e508bc5c7de7d4d0553, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1731341728469 2024-11-11T16:15:29,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:29,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:29,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:29,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:29,383 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:29,384 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/8818bc7e66bb446e93c36b961f8a2050 is 50, key is test_row_0/A:col10/1731341728471/Put/seqid=0 2024-11-11T16:15:29,388 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:29,389 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/936f13df32404ec4b330fa7ab10a5552 is 50, key is test_row_0/B:col10/1731341728471/Put/seqid=0 2024-11-11T16:15:29,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T16:15:29,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,402 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-11T16:15:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:29,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742139_1315 (size=13187) 2024-11-11T16:15:29,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742138_1314 (size=13187) 2024-11-11T16:15:29,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/ae7e444b2fc04f33a89e7ea9f5dfed66 is 50, key is test_row_0/A:col10/1731341728543/Put/seqid=0 2024-11-11T16:15:29,422 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/936f13df32404ec4b330fa7ab10a5552 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/936f13df32404ec4b330fa7ab10a5552 2024-11-11T16:15:29,429 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into 936f13df32404ec4b330fa7ab10a5552(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:29,429 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=12, startTime=1731341729370; duration=0sec 2024-11-11T16:15:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:29,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:29,431 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:29,431 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:29,431 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,431 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/012397aed66e4583b2e583f4c09f0e68, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/17dadf7cd52a49be82c0692a81923f91, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/1ff522536f6a49098ece96720a643d52, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5d90dce36a344289afefeb26a614b694] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=46.4 K 2024-11-11T16:15:29,436 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 012397aed66e4583b2e583f4c09f0e68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=799, earliestPutTs=1731341724960 2024-11-11T16:15:29,436 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 17dadf7cd52a49be82c0692a81923f91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731341726164 2024-11-11T16:15:29,439 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ff522536f6a49098ece96720a643d52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=834, earliestPutTs=1731341726292 2024-11-11T16:15:29,440 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d90dce36a344289afefeb26a614b694, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1731341728469 2024-11-11T16:15:29,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742140_1316 (size=12301) 2024-11-11T16:15:29,441 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/ae7e444b2fc04f33a89e7ea9f5dfed66 2024-11-11T16:15:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T16:15:29,463 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:29,464 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/ea5ba276ce214b5a951e820efee13da7 is 50, key is test_row_0/C:col10/1731341728471/Put/seqid=0 2024-11-11T16:15:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b4515dd3798745d78d85db6f97e10ce9 is 50, key is test_row_0/B:col10/1731341728543/Put/seqid=0 2024-11-11T16:15:29,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742141_1317 (size=13187) 2024-11-11T16:15:29,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742142_1318 (size=12301) 2024-11-11T16:15:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:29,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:29,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341789699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341789705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341789706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341789816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341789816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341789816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:29,824 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/8818bc7e66bb446e93c36b961f8a2050 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8818bc7e66bb446e93c36b961f8a2050 2024-11-11T16:15:29,830 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into 8818bc7e66bb446e93c36b961f8a2050(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:29,830 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:29,830 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=12, startTime=1731341729365; duration=0sec 2024-11-11T16:15:29,830 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:29,830 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:29,906 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/ea5ba276ce214b5a951e820efee13da7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/ea5ba276ce214b5a951e820efee13da7 2024-11-11T16:15:29,916 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into ea5ba276ce214b5a951e820efee13da7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:29,916 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:29,916 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=12, startTime=1731341729380; duration=0sec 2024-11-11T16:15:29,917 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:29,917 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:29,920 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b4515dd3798745d78d85db6f97e10ce9 2024-11-11T16:15:29,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/d366cc9835434fcf85227d3c6d4348da is 50, key is test_row_0/C:col10/1731341728543/Put/seqid=0 2024-11-11T16:15:29,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742143_1319 (size=12301) 2024-11-11T16:15:29,936 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/d366cc9835434fcf85227d3c6d4348da 2024-11-11T16:15:29,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/ae7e444b2fc04f33a89e7ea9f5dfed66 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ae7e444b2fc04f33a89e7ea9f5dfed66 2024-11-11T16:15:29,945 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ae7e444b2fc04f33a89e7ea9f5dfed66, entries=150, sequenceid=880, filesize=12.0 K 2024-11-11T16:15:29,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/b4515dd3798745d78d85db6f97e10ce9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b4515dd3798745d78d85db6f97e10ce9 2024-11-11T16:15:29,951 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b4515dd3798745d78d85db6f97e10ce9, entries=150, sequenceid=880, filesize=12.0 K 2024-11-11T16:15:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/d366cc9835434fcf85227d3c6d4348da as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/d366cc9835434fcf85227d3c6d4348da 2024-11-11T16:15:29,956 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/d366cc9835434fcf85227d3c6d4348da, entries=150, sequenceid=880, filesize=12.0 K 2024-11-11T16:15:29,957 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=53.67 KB/54960 for 8a30e5caa9f1f33bab5e05117c34d967 in 556ms, sequenceid=880, compaction requested=false 2024-11-11T16:15:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T16:15:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T16:15:29,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T16:15:29,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6380 sec 2024-11-11T16:15:29,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6450 sec 2024-11-11T16:15:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:30,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T16:15:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:30,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/da95997518fd48ce8b929112cff7ec8e is 50, key is test_row_0/A:col10/1731341729704/Put/seqid=0 2024-11-11T16:15:30,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742144_1320 (size=14741) 2024-11-11T16:15:30,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731341730050, max timestamp=1731341730064Num uniques 859; 2024-11-11T16:15:30,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731341730050, max timestamp=1731341730064Num uniques 859; 2024-11-11T16:15:30,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:30,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:30,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731341730050, max timestamp=1731341730064Num uniques 859; 2024-11-11T16:15:30,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:30,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341790090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341790096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341790098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341790205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341790206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341790216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341790415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341790415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341790426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=895 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/da95997518fd48ce8b929112cff7ec8e 2024-11-11T16:15:30,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/70c4e3011fc04ec79d153d66b9a6c03b is 50, key is test_row_0/B:col10/1731341729704/Put/seqid=0 2024-11-11T16:15:30,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742145_1321 (size=12301) 2024-11-11T16:15:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T16:15:30,466 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:30,466 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T16:15:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T16:15:30,467 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:30,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:30,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T16:15:30,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T16:15:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341790725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341790725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341790736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:30,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T16:15:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T16:15:30,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=895 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/70c4e3011fc04ec79d153d66b9a6c03b 2024-11-11T16:15:30,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/63a441e7df594486b7a72cce20c6b706 is 50, key is test_row_0/C:col10/1731341729704/Put/seqid=0 2024-11-11T16:15:30,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742146_1322 (size=12301) 2024-11-11T16:15:30,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T16:15:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=895 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/63a441e7df594486b7a72cce20c6b706 2024-11-11T16:15:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:30,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/da95997518fd48ce8b929112cff7ec8e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/da95997518fd48ce8b929112cff7ec8e 2024-11-11T16:15:30,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/da95997518fd48ce8b929112cff7ec8e, entries=200, sequenceid=895, filesize=14.4 K 2024-11-11T16:15:30,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/70c4e3011fc04ec79d153d66b9a6c03b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/70c4e3011fc04ec79d153d66b9a6c03b 2024-11-11T16:15:30,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/70c4e3011fc04ec79d153d66b9a6c03b, entries=150, sequenceid=895, filesize=12.0 K 2024-11-11T16:15:30,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/63a441e7df594486b7a72cce20c6b706 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63a441e7df594486b7a72cce20c6b706 2024-11-11T16:15:30,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63a441e7df594486b7a72cce20c6b706, entries=150, sequenceid=895, filesize=12.0 K 2024-11-11T16:15:30,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 8a30e5caa9f1f33bab5e05117c34d967 in 934ms, sequenceid=895, compaction requested=true 2024-11-11T16:15:30,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:30,964 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:30,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:30,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:30,965 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:30,965 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:30,965 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:30,965 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,965 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8818bc7e66bb446e93c36b961f8a2050, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ae7e444b2fc04f33a89e7ea9f5dfed66, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/da95997518fd48ce8b929112cff7ec8e] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=39.3 K 2024-11-11T16:15:30,966 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8818bc7e66bb446e93c36b961f8a2050, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1731341726292 2024-11-11T16:15:30,966 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:30,966 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:30,966 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:30,966 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/936f13df32404ec4b330fa7ab10a5552, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b4515dd3798745d78d85db6f97e10ce9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/70c4e3011fc04ec79d153d66b9a6c03b] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.9 K 2024-11-11T16:15:30,967 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae7e444b2fc04f33a89e7ea9f5dfed66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=880, earliestPutTs=1731341728518 2024-11-11T16:15:30,967 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 936f13df32404ec4b330fa7ab10a5552, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1731341726292 2024-11-11T16:15:30,967 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b4515dd3798745d78d85db6f97e10ce9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=880, earliestPutTs=1731341728518 2024-11-11T16:15:30,967 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting da95997518fd48ce8b929112cff7ec8e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=895, earliestPutTs=1731341729696 2024-11-11T16:15:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:30,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:30,968 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c4e3011fc04ec79d153d66b9a6c03b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=895, earliestPutTs=1731341729697 2024-11-11T16:15:30,983 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:30,983 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/8b3c142dfac94b6096371fb23ef7ce80 is 50, key is test_row_0/B:col10/1731341729704/Put/seqid=0 2024-11-11T16:15:30,986 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:30,987 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/1dde71ec079549508aa4a9a0930c32c8 is 50, key is test_row_0/A:col10/1731341729704/Put/seqid=0 2024-11-11T16:15:31,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742147_1323 (size=13289) 2024-11-11T16:15:31,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742148_1324 (size=13289) 2024-11-11T16:15:31,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T16:15:31,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:31,087 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-11T16:15:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:31,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/8c8822c5994b4518ad2046bbd1a4c0e7 is 50, key is test_row_0/A:col10/1731341730094/Put/seqid=0 2024-11-11T16:15:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T16:15:31,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742149_1325 (size=12301) 2024-11-11T16:15:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:31,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341791254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341791254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341791255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341791265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38114 deadline: 1731341791265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,269 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 8a30e5caa9f1f33bab5e05117c34d967 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341791366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341791366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341791376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341791376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,415 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/1dde71ec079549508aa4a9a0930c32c8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/1dde71ec079549508aa4a9a0930c32c8 2024-11-11T16:15:31,417 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/8b3c142dfac94b6096371fb23ef7ce80 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/8b3c142dfac94b6096371fb23ef7ce80 2024-11-11T16:15:31,425 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into 1dde71ec079549508aa4a9a0930c32c8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:31,425 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into 8b3c142dfac94b6096371fb23ef7ce80(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:31,425 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:31,425 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:31,425 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341730965; duration=0sec 2024-11-11T16:15:31,425 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341730964; duration=0sec 2024-11-11T16:15:31,426 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:31,426 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:31,426 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:31,426 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:31,426 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:31,427 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:31,427 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:31,427 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:31,427 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/ea5ba276ce214b5a951e820efee13da7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/d366cc9835434fcf85227d3c6d4348da, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63a441e7df594486b7a72cce20c6b706] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=36.9 K 2024-11-11T16:15:31,428 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ea5ba276ce214b5a951e820efee13da7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1731341726292 2024-11-11T16:15:31,428 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d366cc9835434fcf85227d3c6d4348da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=880, earliestPutTs=1731341728518 2024-11-11T16:15:31,429 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a441e7df594486b7a72cce20c6b706, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=895, earliestPutTs=1731341729697 2024-11-11T16:15:31,435 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:31,437 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/3d899a18e7084608b8e5ac42569d393f is 50, key is test_row_0/C:col10/1731341729704/Put/seqid=0 2024-11-11T16:15:31,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742150_1326 (size=13289) 2024-11-11T16:15:31,498 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/8c8822c5994b4518ad2046bbd1a4c0e7 2024-11-11T16:15:31,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/531a49272c3c4ebf8e6bcdc21edc6576 is 50, key is test_row_0/B:col10/1731341730094/Put/seqid=0 2024-11-11T16:15:31,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742151_1327 (size=12301) 2024-11-11T16:15:31,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341791576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341791576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341791585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341791586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T16:15:31,859 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/3d899a18e7084608b8e5ac42569d393f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3d899a18e7084608b8e5ac42569d393f 2024-11-11T16:15:31,863 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into 3d899a18e7084608b8e5ac42569d393f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:31,863 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:31,863 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341730968; duration=0sec 2024-11-11T16:15:31,864 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:31,864 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:31,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341791886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341791886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341791895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341791895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:31,920 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/531a49272c3c4ebf8e6bcdc21edc6576 2024-11-11T16:15:31,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/6fd102919e6642b19d3dd85e46926e63 is 50, key is test_row_0/C:col10/1731341730094/Put/seqid=0 2024-11-11T16:15:31,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742152_1328 (size=12301) 2024-11-11T16:15:32,333 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=928 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/6fd102919e6642b19d3dd85e46926e63 2024-11-11T16:15:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/8c8822c5994b4518ad2046bbd1a4c0e7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8c8822c5994b4518ad2046bbd1a4c0e7 2024-11-11T16:15:32,350 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8c8822c5994b4518ad2046bbd1a4c0e7, entries=150, sequenceid=928, filesize=12.0 K 2024-11-11T16:15:32,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/531a49272c3c4ebf8e6bcdc21edc6576 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/531a49272c3c4ebf8e6bcdc21edc6576 2024-11-11T16:15:32,356 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/531a49272c3c4ebf8e6bcdc21edc6576, entries=150, sequenceid=928, filesize=12.0 K 2024-11-11T16:15:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/6fd102919e6642b19d3dd85e46926e63 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/6fd102919e6642b19d3dd85e46926e63 2024-11-11T16:15:32,364 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/6fd102919e6642b19d3dd85e46926e63, entries=150, sequenceid=928, filesize=12.0 K 2024-11-11T16:15:32,364 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for 8a30e5caa9f1f33bab5e05117c34d967 in 1277ms, sequenceid=928, compaction requested=false 2024-11-11T16:15:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T16:15:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T16:15:32,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T16:15:32,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8990 sec 2024-11-11T16:15:32,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.9040 sec 2024-11-11T16:15:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:32,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:15:32,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:32,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/fe8715c467d64b3182f4cb51b65b8906 is 50, key is test_row_0/A:col10/1731341731253/Put/seqid=0 2024-11-11T16:15:32,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742153_1329 (size=12301) 2024-11-11T16:15:32,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341792428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341792428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341792433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341792434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341792546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341792546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341792546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341792546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T16:15:32,616 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:32,616 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T16:15:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T16:15:32,618 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:32,619 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:32,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T16:15:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341792756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341792756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341792756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341792756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:32,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T16:15:32,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:32,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:32,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:32,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:32,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=944 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/fe8715c467d64b3182f4cb51b65b8906 2024-11-11T16:15:32,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/32149aaf850644528faf3128ef69a1f8 is 50, key is test_row_0/B:col10/1731341731253/Put/seqid=0 2024-11-11T16:15:32,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742154_1330 (size=12301) 2024-11-11T16:15:32,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T16:15:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:32,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:32,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:32,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:32,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T16:15:33,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341793066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341793066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341793066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341793066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T16:15:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=944 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/32149aaf850644528faf3128ef69a1f8 2024-11-11T16:15:33,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T16:15:33,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:33,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/63ba865b0bad488e87290438c6d16ffa is 50, key is test_row_0/C:col10/1731341731253/Put/seqid=0 2024-11-11T16:15:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T16:15:33,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742155_1331 (size=12301) 2024-11-11T16:15:33,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T16:15:33,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:33,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T16:15:33,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. as already flushing 2024-11-11T16:15:33,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:33,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1731341793576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1731341793576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38070 deadline: 1731341793576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38162 deadline: 1731341793586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:15:33,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=944 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/63ba865b0bad488e87290438c6d16ffa 2024-11-11T16:15:33,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/fe8715c467d64b3182f4cb51b65b8906 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fe8715c467d64b3182f4cb51b65b8906 2024-11-11T16:15:33,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fe8715c467d64b3182f4cb51b65b8906, entries=150, sequenceid=944, filesize=12.0 K 2024-11-11T16:15:33,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/32149aaf850644528faf3128ef69a1f8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/32149aaf850644528faf3128ef69a1f8 2024-11-11T16:15:33,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/32149aaf850644528faf3128ef69a1f8, entries=150, sequenceid=944, filesize=12.0 K 2024-11-11T16:15:33,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/63ba865b0bad488e87290438c6d16ffa as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63ba865b0bad488e87290438c6d16ffa 2024-11-11T16:15:33,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63ba865b0bad488e87290438c6d16ffa, entries=150, sequenceid=944, filesize=12.0 K 2024-11-11T16:15:33,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8a30e5caa9f1f33bab5e05117c34d967 in 1287ms, sequenceid=944, compaction requested=true 2024-11-11T16:15:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:33,685 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a30e5caa9f1f33bab5e05117c34d967:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:33,685 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:33,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:33,686 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:33,686 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/A is initiating minor compaction (all files) 2024-11-11T16:15:33,686 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/A in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,686 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/1dde71ec079549508aa4a9a0930c32c8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8c8822c5994b4518ad2046bbd1a4c0e7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fe8715c467d64b3182f4cb51b65b8906] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=37.0 K 2024-11-11T16:15:33,686 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:33,687 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/B is initiating minor compaction (all files) 2024-11-11T16:15:33,687 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/B in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,687 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/8b3c142dfac94b6096371fb23ef7ce80, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/531a49272c3c4ebf8e6bcdc21edc6576, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/32149aaf850644528faf3128ef69a1f8] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=37.0 K 2024-11-11T16:15:33,687 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dde71ec079549508aa4a9a0930c32c8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=895, earliestPutTs=1731341729697 2024-11-11T16:15:33,687 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b3c142dfac94b6096371fb23ef7ce80, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=895, earliestPutTs=1731341729697 2024-11-11T16:15:33,687 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8822c5994b4518ad2046bbd1a4c0e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=928, earliestPutTs=1731341730089 2024-11-11T16:15:33,688 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 531a49272c3c4ebf8e6bcdc21edc6576, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=928, earliestPutTs=1731341730089 2024-11-11T16:15:33,688 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe8715c467d64b3182f4cb51b65b8906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=944, earliestPutTs=1731341731252 2024-11-11T16:15:33,688 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 32149aaf850644528faf3128ef69a1f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=944, earliestPutTs=1731341731252 2024-11-11T16:15:33,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T16:15:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:33,691 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:15:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:33,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:33,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:33,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d91685203ef7462b968643023f651658 is 50, key is test_row_0/A:col10/1731341732425/Put/seqid=0 2024-11-11T16:15:33,696 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:33,697 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4d75f5a2406c48f09e867bed3acf65a1 is 50, key is test_row_0/B:col10/1731341731253/Put/seqid=0 2024-11-11T16:15:33,702 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#A#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:33,702 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/daaa29eeca064836b4947e1fea2cdf8b is 50, key is test_row_0/A:col10/1731341731253/Put/seqid=0 2024-11-11T16:15:33,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742156_1332 (size=12301) 2024-11-11T16:15:33,719 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=963 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d91685203ef7462b968643023f651658 2024-11-11T16:15:33,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e9bde28afecb467b9a223df9a59a52fa is 50, key is test_row_0/B:col10/1731341732425/Put/seqid=0 2024-11-11T16:15:33,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742158_1334 (size=13391) 2024-11-11T16:15:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742157_1333 (size=13391) 2024-11-11T16:15:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742159_1335 (size=12301) 2024-11-11T16:15:33,748 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=963 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e9bde28afecb467b9a223df9a59a52fa 2024-11-11T16:15:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T16:15:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/7ad3bebc7d9e42b3bad688e1ad158335 is 50, key is test_row_0/C:col10/1731341732425/Put/seqid=0 2024-11-11T16:15:33,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742160_1336 (size=12301) 2024-11-11T16:15:34,142 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4d75f5a2406c48f09e867bed3acf65a1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4d75f5a2406c48f09e867bed3acf65a1 2024-11-11T16:15:34,142 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/daaa29eeca064836b4947e1fea2cdf8b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/daaa29eeca064836b4947e1fea2cdf8b 2024-11-11T16:15:34,147 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/A of 8a30e5caa9f1f33bab5e05117c34d967 into daaa29eeca064836b4947e1fea2cdf8b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:34,148 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:34,148 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/A, priority=13, startTime=1731341733685; duration=0sec 2024-11-11T16:15:34,148 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:34,148 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:A 2024-11-11T16:15:34,148 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:34,149 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/B of 8a30e5caa9f1f33bab5e05117c34d967 into 4d75f5a2406c48f09e867bed3acf65a1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:34,149 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:34,149 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/B, priority=13, startTime=1731341733685; duration=0sec 2024-11-11T16:15:34,149 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:34,149 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:B 2024-11-11T16:15:34,150 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:34,150 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 8a30e5caa9f1f33bab5e05117c34d967/C is initiating minor compaction (all files) 2024-11-11T16:15:34,150 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a30e5caa9f1f33bab5e05117c34d967/C in TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:34,150 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3d899a18e7084608b8e5ac42569d393f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/6fd102919e6642b19d3dd85e46926e63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63ba865b0bad488e87290438c6d16ffa] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp, totalSize=37.0 K 2024-11-11T16:15:34,150 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d899a18e7084608b8e5ac42569d393f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=895, earliestPutTs=1731341729697 2024-11-11T16:15:34,151 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fd102919e6642b19d3dd85e46926e63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=928, earliestPutTs=1731341730089 2024-11-11T16:15:34,151 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63ba865b0bad488e87290438c6d16ffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=944, earliestPutTs=1731341731252 2024-11-11T16:15:34,159 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a30e5caa9f1f33bab5e05117c34d967#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:34,159 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e6719663009849c89dabab77ce2dc46f is 50, key is test_row_0/C:col10/1731341731253/Put/seqid=0 2024-11-11T16:15:34,175 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=963 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/7ad3bebc7d9e42b3bad688e1ad158335 2024-11-11T16:15:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d91685203ef7462b968643023f651658 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d91685203ef7462b968643023f651658 2024-11-11T16:15:34,186 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d91685203ef7462b968643023f651658, entries=150, sequenceid=963, filesize=12.0 K 2024-11-11T16:15:34,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/e9bde28afecb467b9a223df9a59a52fa as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e9bde28afecb467b9a223df9a59a52fa 2024-11-11T16:15:34,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742161_1337 (size=13391) 2024-11-11T16:15:34,197 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e9bde28afecb467b9a223df9a59a52fa, entries=150, sequenceid=963, filesize=12.0 K 2024-11-11T16:15:34,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/7ad3bebc7d9e42b3bad688e1ad158335 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/7ad3bebc7d9e42b3bad688e1ad158335 2024-11-11T16:15:34,199 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/e6719663009849c89dabab77ce2dc46f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e6719663009849c89dabab77ce2dc46f 2024-11-11T16:15:34,204 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/7ad3bebc7d9e42b3bad688e1ad158335, entries=150, sequenceid=963, filesize=12.0 K 2024-11-11T16:15:34,204 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a30e5caa9f1f33bab5e05117c34d967/C of 8a30e5caa9f1f33bab5e05117c34d967 into e6719663009849c89dabab77ce2dc46f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:34,204 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:34,204 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., storeName=8a30e5caa9f1f33bab5e05117c34d967/C, priority=13, startTime=1731341733685; duration=0sec 2024-11-11T16:15:34,204 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:34,204 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a30e5caa9f1f33bab5e05117c34d967:C 2024-11-11T16:15:34,205 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 8a30e5caa9f1f33bab5e05117c34d967 in 514ms, sequenceid=963, compaction requested=false 2024-11-11T16:15:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8a30e5caa9f1f33bab5e05117c34d967: 2024-11-11T16:15:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T16:15:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T16:15:34,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T16:15:34,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-11-11T16:15:34,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5950 sec 2024-11-11T16:15:34,433 INFO [Thread-1116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1116. 2024-11-11T16:15:34,433 DEBUG [Thread-1116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,433 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,433 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-11T16:15:34,433 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,433 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,433 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,433 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-11T16:15:34,433 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-11T16:15:34,433 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,433 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,433 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,433 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,433 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,433 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,434 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,435 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-11T16:15:34,435 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,435 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,435 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,588 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-11T16:15:34,588 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,588 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,588 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,588 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,597 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-11T16:15:34,597 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,597 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,597 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-11T16:15:34,597 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,597 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,597 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,598 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,607 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-11T16:15:34,607 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:34,607 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,607 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:34,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T16:15:34,766 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:36,365 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T16:15:41,327 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-11T16:15:41,327 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:15:41,327 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:41,327 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:41,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 205 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 216 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 205 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2371 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2339 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2471 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2358 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2361 2024-11-11T16:15:41,327 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T16:15:41,327 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:15:41,327 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:15:41,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:41,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:41,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:15:41,328 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T16:15:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T16:15:41,331 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341741331"}]},"ts":"1731341741331"} 2024-11-11T16:15:41,332 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T16:15:41,332 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T16:15:41,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:15:41,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, UNASSIGN}] 2024-11-11T16:15:41,335 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, UNASSIGN 2024-11-11T16:15:41,335 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=8a30e5caa9f1f33bab5e05117c34d967, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:41,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, UNASSIGN because future has completed 2024-11-11T16:15:41,336 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:15:41,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:15:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T16:15:41,487 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 8a30e5caa9f1f33bab5e05117c34d967, disabling compactions & flushes 2024-11-11T16:15:41,488 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. after waiting 0 ms 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:41,488 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 8a30e5caa9f1f33bab5e05117c34d967 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=A 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=B 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a30e5caa9f1f33bab5e05117c34d967, store=C 2024-11-11T16:15:41,488 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:41,492 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d6b0d739102548be8da0a043837887e2 is 50, key is test_row_0/A:col10/1731341734606/Put/seqid=0 2024-11-11T16:15:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742162_1338 (size=12301) 2024-11-11T16:15:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T16:15:41,896 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=974 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d6b0d739102548be8da0a043837887e2 2024-11-11T16:15:41,902 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4319af3b4ee541bfab275242e5af0759 is 50, key is test_row_0/B:col10/1731341734606/Put/seqid=0 2024-11-11T16:15:41,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742163_1339 (size=12301) 2024-11-11T16:15:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T16:15:42,306 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=974 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4319af3b4ee541bfab275242e5af0759 2024-11-11T16:15:42,312 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/de46993825a042b1ac8468a2e6aaa7ac is 50, key is test_row_0/C:col10/1731341734606/Put/seqid=0 2024-11-11T16:15:42,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742164_1340 (size=12301) 2024-11-11T16:15:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T16:15:42,716 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=974 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/de46993825a042b1ac8468a2e6aaa7ac 2024-11-11T16:15:42,720 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/A/d6b0d739102548be8da0a043837887e2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d6b0d739102548be8da0a043837887e2 2024-11-11T16:15:42,723 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d6b0d739102548be8da0a043837887e2, entries=150, sequenceid=974, filesize=12.0 K 2024-11-11T16:15:42,724 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/B/4319af3b4ee541bfab275242e5af0759 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4319af3b4ee541bfab275242e5af0759 2024-11-11T16:15:42,727 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4319af3b4ee541bfab275242e5af0759, entries=150, sequenceid=974, filesize=12.0 K 2024-11-11T16:15:42,728 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/.tmp/C/de46993825a042b1ac8468a2e6aaa7ac as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/de46993825a042b1ac8468a2e6aaa7ac 2024-11-11T16:15:42,731 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/de46993825a042b1ac8468a2e6aaa7ac, entries=150, sequenceid=974, filesize=12.0 K 2024-11-11T16:15:42,732 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8a30e5caa9f1f33bab5e05117c34d967 in 1243ms, sequenceid=974, compaction requested=true 2024-11-11T16:15:42,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ad7f5ce26100423e90f6e31b4a5cb6a2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c412a9a941744691bf641da5df604ec6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/9b4ec9727ccd4a8da443724fe16cbd45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/43834f3144b94d4e87e39609a042df87, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/584b29ce8376499dbf8eec3cecbdb82a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/42ff5a7e4b8d463bb6bce1ce4842d5ad, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c500ffa8b7e34fc8866fceb143cdba67, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/7b3bc2ff42384ad0bf6acdf3e55d9679, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3c359511f11447a5abe731e2bd4d9245, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/56442c328b71408ab70d82f724759a54, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8f0c57479584eb4898e2df38157b926, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/bdaab3b0f64f4605982b69b1422fdf36, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/6ed8f41df0104812af894b3842eb9399, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/13ec71476ec649e49a73061448e1beab, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/af712e9f23924e43a58fc6a89936e4e4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c8cfa0ac4bcb428eb37062c78df3fe10, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/34486384813943b8a256d2ccc4318110, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/31e4b60fe4db44039b1aed5719c257bc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8697d9cdc0c43b6998c6dba24ef7e5d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b2c0edd9dddc47ab8e16f8b0033f32fa, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/727f91880faa443a869271b730731527, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fad477392685431f82524dd62c414e59, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/841093c93b3b45fab2f78aa41750ea9c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b3c53bc6aac84e43a142c3b9be69f7e3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/f6fd47d7ec1c44749859187a77fff4d4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8818bc7e66bb446e93c36b961f8a2050, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3175689001ff425da8dda53e82a5e627, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ae7e444b2fc04f33a89e7ea9f5dfed66, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/da95997518fd48ce8b929112cff7ec8e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/1dde71ec079549508aa4a9a0930c32c8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8c8822c5994b4518ad2046bbd1a4c0e7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fe8715c467d64b3182f4cb51b65b8906] to archive 2024-11-11T16:15:42,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:15:42,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ad7f5ce26100423e90f6e31b4a5cb6a2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ad7f5ce26100423e90f6e31b4a5cb6a2 2024-11-11T16:15:42,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c412a9a941744691bf641da5df604ec6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c412a9a941744691bf641da5df604ec6 2024-11-11T16:15:42,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/9b4ec9727ccd4a8da443724fe16cbd45 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/9b4ec9727ccd4a8da443724fe16cbd45 2024-11-11T16:15:42,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/43834f3144b94d4e87e39609a042df87 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/43834f3144b94d4e87e39609a042df87 2024-11-11T16:15:42,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/584b29ce8376499dbf8eec3cecbdb82a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/584b29ce8376499dbf8eec3cecbdb82a 2024-11-11T16:15:42,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/42ff5a7e4b8d463bb6bce1ce4842d5ad to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/42ff5a7e4b8d463bb6bce1ce4842d5ad 2024-11-11T16:15:42,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c500ffa8b7e34fc8866fceb143cdba67 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c500ffa8b7e34fc8866fceb143cdba67 2024-11-11T16:15:42,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/7b3bc2ff42384ad0bf6acdf3e55d9679 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/7b3bc2ff42384ad0bf6acdf3e55d9679 2024-11-11T16:15:42,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3c359511f11447a5abe731e2bd4d9245 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3c359511f11447a5abe731e2bd4d9245 2024-11-11T16:15:42,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/56442c328b71408ab70d82f724759a54 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/56442c328b71408ab70d82f724759a54 2024-11-11T16:15:42,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8f0c57479584eb4898e2df38157b926 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8f0c57479584eb4898e2df38157b926 2024-11-11T16:15:42,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/bdaab3b0f64f4605982b69b1422fdf36 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/bdaab3b0f64f4605982b69b1422fdf36 2024-11-11T16:15:42,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/6ed8f41df0104812af894b3842eb9399 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/6ed8f41df0104812af894b3842eb9399 2024-11-11T16:15:42,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/13ec71476ec649e49a73061448e1beab to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/13ec71476ec649e49a73061448e1beab 2024-11-11T16:15:42,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/af712e9f23924e43a58fc6a89936e4e4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/af712e9f23924e43a58fc6a89936e4e4 2024-11-11T16:15:42,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c8cfa0ac4bcb428eb37062c78df3fe10 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/c8cfa0ac4bcb428eb37062c78df3fe10 2024-11-11T16:15:42,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/34486384813943b8a256d2ccc4318110 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/34486384813943b8a256d2ccc4318110 2024-11-11T16:15:42,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/31e4b60fe4db44039b1aed5719c257bc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/31e4b60fe4db44039b1aed5719c257bc 2024-11-11T16:15:42,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8697d9cdc0c43b6998c6dba24ef7e5d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d8697d9cdc0c43b6998c6dba24ef7e5d 2024-11-11T16:15:42,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b2c0edd9dddc47ab8e16f8b0033f32fa to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b2c0edd9dddc47ab8e16f8b0033f32fa 2024-11-11T16:15:42,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/727f91880faa443a869271b730731527 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/727f91880faa443a869271b730731527 2024-11-11T16:15:42,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fad477392685431f82524dd62c414e59 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fad477392685431f82524dd62c414e59 2024-11-11T16:15:42,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/841093c93b3b45fab2f78aa41750ea9c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/841093c93b3b45fab2f78aa41750ea9c 2024-11-11T16:15:42,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b3c53bc6aac84e43a142c3b9be69f7e3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/b3c53bc6aac84e43a142c3b9be69f7e3 2024-11-11T16:15:42,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/f6fd47d7ec1c44749859187a77fff4d4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/f6fd47d7ec1c44749859187a77fff4d4 2024-11-11T16:15:42,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8818bc7e66bb446e93c36b961f8a2050 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8818bc7e66bb446e93c36b961f8a2050 2024-11-11T16:15:42,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3175689001ff425da8dda53e82a5e627 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/3175689001ff425da8dda53e82a5e627 2024-11-11T16:15:42,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ae7e444b2fc04f33a89e7ea9f5dfed66 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/ae7e444b2fc04f33a89e7ea9f5dfed66 2024-11-11T16:15:42,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/da95997518fd48ce8b929112cff7ec8e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/da95997518fd48ce8b929112cff7ec8e 2024-11-11T16:15:42,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/1dde71ec079549508aa4a9a0930c32c8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/1dde71ec079549508aa4a9a0930c32c8 2024-11-11T16:15:42,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8c8822c5994b4518ad2046bbd1a4c0e7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/8c8822c5994b4518ad2046bbd1a4c0e7 2024-11-11T16:15:42,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fe8715c467d64b3182f4cb51b65b8906 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/fe8715c467d64b3182f4cb51b65b8906 2024-11-11T16:15:42,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/aebe38170a354a998d5d58732b2b4a09, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/84459d028c374ef5af936eea1e33f9d0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b3d86e1cf8024a6f81f48bf0fd1b8eb7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/9cdbc7c5fb454e74a55459cd79da0651, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/018144469a0c4eb09882a47a1ab77d2e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0db7166dc35548039e4b7e77a48f0c90, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0fc5b7336e244f538cea735e147184a6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d93ce57d9ab847aabcdbf16297eb577a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/5ba08f8774904763b73c372296da1c65, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/2cd6fed8634d4fffba691485f14bff84, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d367402bae524481a50f8f4f2e71c641, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/23b462a92c3f4c449fb9f138104d184b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/a55782e168d147f5b836194ec64220e7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4eeedaf233fa446e8134986b8d824817, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/65f3ee35c8364e17baaaf6f857b78066, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e5a82f3054294ce9bbca69e155e75109, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/7c14332c3619492588532817ba9e64f8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/f0b70d97eae048f181bb86e0dd8f6265, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/ede9e510a706428c9869174bc39664f5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e002be85d1ba44e584c81265236631da, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/59acb941b69b4dd186e8491605ec05cc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/02ac2a70f3d84efd9684be5b238e6856, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b9d7d8ebed35473c98f8ee42c885c885, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/24aed262011442648df9ac616706005b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/bc1bec641f4847798f9a3d9eedfc5b3b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/936f13df32404ec4b330fa7ab10a5552, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/34311c7462184e508bc5c7de7d4d0553, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b4515dd3798745d78d85db6f97e10ce9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/8b3c142dfac94b6096371fb23ef7ce80, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/70c4e3011fc04ec79d153d66b9a6c03b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/531a49272c3c4ebf8e6bcdc21edc6576, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/32149aaf850644528faf3128ef69a1f8] to archive 2024-11-11T16:15:42,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:15:42,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/aebe38170a354a998d5d58732b2b4a09 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/aebe38170a354a998d5d58732b2b4a09 2024-11-11T16:15:42,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/84459d028c374ef5af936eea1e33f9d0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/84459d028c374ef5af936eea1e33f9d0 2024-11-11T16:15:42,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b3d86e1cf8024a6f81f48bf0fd1b8eb7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b3d86e1cf8024a6f81f48bf0fd1b8eb7 2024-11-11T16:15:42,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/9cdbc7c5fb454e74a55459cd79da0651 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/9cdbc7c5fb454e74a55459cd79da0651 2024-11-11T16:15:42,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/018144469a0c4eb09882a47a1ab77d2e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/018144469a0c4eb09882a47a1ab77d2e 2024-11-11T16:15:42,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0db7166dc35548039e4b7e77a48f0c90 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0db7166dc35548039e4b7e77a48f0c90 2024-11-11T16:15:42,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0fc5b7336e244f538cea735e147184a6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/0fc5b7336e244f538cea735e147184a6 2024-11-11T16:15:42,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d93ce57d9ab847aabcdbf16297eb577a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d93ce57d9ab847aabcdbf16297eb577a 2024-11-11T16:15:42,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/5ba08f8774904763b73c372296da1c65 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/5ba08f8774904763b73c372296da1c65 2024-11-11T16:15:42,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/2cd6fed8634d4fffba691485f14bff84 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/2cd6fed8634d4fffba691485f14bff84 2024-11-11T16:15:42,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d367402bae524481a50f8f4f2e71c641 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/d367402bae524481a50f8f4f2e71c641 2024-11-11T16:15:42,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/23b462a92c3f4c449fb9f138104d184b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/23b462a92c3f4c449fb9f138104d184b 2024-11-11T16:15:42,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/a55782e168d147f5b836194ec64220e7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/a55782e168d147f5b836194ec64220e7 2024-11-11T16:15:42,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4eeedaf233fa446e8134986b8d824817 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4eeedaf233fa446e8134986b8d824817 2024-11-11T16:15:42,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/65f3ee35c8364e17baaaf6f857b78066 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/65f3ee35c8364e17baaaf6f857b78066 2024-11-11T16:15:42,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e5a82f3054294ce9bbca69e155e75109 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e5a82f3054294ce9bbca69e155e75109 2024-11-11T16:15:42,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/7c14332c3619492588532817ba9e64f8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/7c14332c3619492588532817ba9e64f8 2024-11-11T16:15:42,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/f0b70d97eae048f181bb86e0dd8f6265 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/f0b70d97eae048f181bb86e0dd8f6265 2024-11-11T16:15:42,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/ede9e510a706428c9869174bc39664f5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/ede9e510a706428c9869174bc39664f5 2024-11-11T16:15:42,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e002be85d1ba44e584c81265236631da to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e002be85d1ba44e584c81265236631da 2024-11-11T16:15:42,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/59acb941b69b4dd186e8491605ec05cc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/59acb941b69b4dd186e8491605ec05cc 2024-11-11T16:15:42,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/02ac2a70f3d84efd9684be5b238e6856 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/02ac2a70f3d84efd9684be5b238e6856 2024-11-11T16:15:42,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b9d7d8ebed35473c98f8ee42c885c885 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b9d7d8ebed35473c98f8ee42c885c885 2024-11-11T16:15:42,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/24aed262011442648df9ac616706005b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/24aed262011442648df9ac616706005b 2024-11-11T16:15:42,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/bc1bec641f4847798f9a3d9eedfc5b3b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/bc1bec641f4847798f9a3d9eedfc5b3b 2024-11-11T16:15:42,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/936f13df32404ec4b330fa7ab10a5552 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/936f13df32404ec4b330fa7ab10a5552 2024-11-11T16:15:42,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/34311c7462184e508bc5c7de7d4d0553 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/34311c7462184e508bc5c7de7d4d0553 2024-11-11T16:15:42,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b4515dd3798745d78d85db6f97e10ce9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/b4515dd3798745d78d85db6f97e10ce9 2024-11-11T16:15:42,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/8b3c142dfac94b6096371fb23ef7ce80 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/8b3c142dfac94b6096371fb23ef7ce80 2024-11-11T16:15:42,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/70c4e3011fc04ec79d153d66b9a6c03b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/70c4e3011fc04ec79d153d66b9a6c03b 2024-11-11T16:15:42,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/531a49272c3c4ebf8e6bcdc21edc6576 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/531a49272c3c4ebf8e6bcdc21edc6576 2024-11-11T16:15:42,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/32149aaf850644528faf3128ef69a1f8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/32149aaf850644528faf3128ef69a1f8 2024-11-11T16:15:42,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/2bd21fb4684848989a7c73be5032db05, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e9b921a4a8a44865a587a6fbffc3642d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/28871775dd664610bc869788d917d2c0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3ba349969d794acc9ed646ba7c2665a4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/bcde75f8dc8c4331a06737b7c818c986, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5299e2d34acf46a68ba2d8e23d59cfc5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/0607b13cc08741e48c3547457ddcdec7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c1346835a1c4a25ae0984f5b1e8e7fc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/a4ac68b29a8f48b296394bbe50a7e39b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/8c8ea86d38b7440281610d9011b487fe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c42d6c6d3254642a45b7fb508b5f016, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/da5d6e88a9f044f1a62b57e31b35bf4d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/33c214ed2bb84e7da4548dc018736332, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e75c642c7a714e62875e65db5ac0dd60, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e955fa36654e413586dacb2788fc8e0d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b4a62678a6f1451cbbef41e56e0c430f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/12a3dbdce5624a3aa2e3bec6a34d4141, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b57406d0c6d94f5e963bb399fac4a517, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/14d5bd1979074a79b35aeb9f4a34aad0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/012397aed66e4583b2e583f4c09f0e68, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/83cf328957a9471c9177f6f23a932c4e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/17dadf7cd52a49be82c0692a81923f91, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/1ff522536f6a49098ece96720a643d52, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/ea5ba276ce214b5a951e820efee13da7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5d90dce36a344289afefeb26a614b694, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/d366cc9835434fcf85227d3c6d4348da, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3d899a18e7084608b8e5ac42569d393f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63a441e7df594486b7a72cce20c6b706, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/6fd102919e6642b19d3dd85e46926e63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63ba865b0bad488e87290438c6d16ffa] to archive 2024-11-11T16:15:42,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:15:42,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/2bd21fb4684848989a7c73be5032db05 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/2bd21fb4684848989a7c73be5032db05 2024-11-11T16:15:42,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e9b921a4a8a44865a587a6fbffc3642d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e9b921a4a8a44865a587a6fbffc3642d 2024-11-11T16:15:42,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/28871775dd664610bc869788d917d2c0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/28871775dd664610bc869788d917d2c0 2024-11-11T16:15:42,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/32c5e7ba2c7d4509bc74ca4dbcbe1dca 2024-11-11T16:15:42,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3ba349969d794acc9ed646ba7c2665a4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3ba349969d794acc9ed646ba7c2665a4 2024-11-11T16:15:42,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/bcde75f8dc8c4331a06737b7c818c986 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/bcde75f8dc8c4331a06737b7c818c986 2024-11-11T16:15:42,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5299e2d34acf46a68ba2d8e23d59cfc5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5299e2d34acf46a68ba2d8e23d59cfc5 2024-11-11T16:15:42,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/0607b13cc08741e48c3547457ddcdec7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/0607b13cc08741e48c3547457ddcdec7 2024-11-11T16:15:42,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c1346835a1c4a25ae0984f5b1e8e7fc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c1346835a1c4a25ae0984f5b1e8e7fc 2024-11-11T16:15:42,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/a4ac68b29a8f48b296394bbe50a7e39b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/a4ac68b29a8f48b296394bbe50a7e39b 2024-11-11T16:15:42,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/8c8ea86d38b7440281610d9011b487fe to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/8c8ea86d38b7440281610d9011b487fe 2024-11-11T16:15:42,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c42d6c6d3254642a45b7fb508b5f016 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/4c42d6c6d3254642a45b7fb508b5f016 2024-11-11T16:15:42,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/da5d6e88a9f044f1a62b57e31b35bf4d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/da5d6e88a9f044f1a62b57e31b35bf4d 2024-11-11T16:15:42,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/33c214ed2bb84e7da4548dc018736332 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/33c214ed2bb84e7da4548dc018736332 2024-11-11T16:15:42,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/53cc78f6e5784a0fb2bc1b3b1a6c6bb2 2024-11-11T16:15:42,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e75c642c7a714e62875e65db5ac0dd60 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e75c642c7a714e62875e65db5ac0dd60 2024-11-11T16:15:42,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e955fa36654e413586dacb2788fc8e0d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e955fa36654e413586dacb2788fc8e0d 2024-11-11T16:15:42,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b4a62678a6f1451cbbef41e56e0c430f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b4a62678a6f1451cbbef41e56e0c430f 2024-11-11T16:15:42,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/12a3dbdce5624a3aa2e3bec6a34d4141 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/12a3dbdce5624a3aa2e3bec6a34d4141 2024-11-11T16:15:42,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b57406d0c6d94f5e963bb399fac4a517 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/b57406d0c6d94f5e963bb399fac4a517 2024-11-11T16:15:42,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/14d5bd1979074a79b35aeb9f4a34aad0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/14d5bd1979074a79b35aeb9f4a34aad0 2024-11-11T16:15:42,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/012397aed66e4583b2e583f4c09f0e68 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/012397aed66e4583b2e583f4c09f0e68 2024-11-11T16:15:42,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/83cf328957a9471c9177f6f23a932c4e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/83cf328957a9471c9177f6f23a932c4e 2024-11-11T16:15:42,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/17dadf7cd52a49be82c0692a81923f91 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/17dadf7cd52a49be82c0692a81923f91 2024-11-11T16:15:42,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/1ff522536f6a49098ece96720a643d52 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/1ff522536f6a49098ece96720a643d52 2024-11-11T16:15:42,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/ea5ba276ce214b5a951e820efee13da7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/ea5ba276ce214b5a951e820efee13da7 2024-11-11T16:15:42,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5d90dce36a344289afefeb26a614b694 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/5d90dce36a344289afefeb26a614b694 2024-11-11T16:15:42,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/d366cc9835434fcf85227d3c6d4348da to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/d366cc9835434fcf85227d3c6d4348da 2024-11-11T16:15:42,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3d899a18e7084608b8e5ac42569d393f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/3d899a18e7084608b8e5ac42569d393f 2024-11-11T16:15:42,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63a441e7df594486b7a72cce20c6b706 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63a441e7df594486b7a72cce20c6b706 2024-11-11T16:15:42,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/6fd102919e6642b19d3dd85e46926e63 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/6fd102919e6642b19d3dd85e46926e63 2024-11-11T16:15:42,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63ba865b0bad488e87290438c6d16ffa to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/63ba865b0bad488e87290438c6d16ffa 2024-11-11T16:15:42,844 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/recovered.edits/977.seqid, newMaxSeqId=977, maxSeqId=1 2024-11-11T16:15:42,845 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. 2024-11-11T16:15:42,845 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 8a30e5caa9f1f33bab5e05117c34d967: Waiting for close lock at 1731341741488Running coprocessor pre-close hooks at 1731341741488Disabling compacts and flushes for region at 1731341741488Disabling writes for close at 1731341741488Obtaining lock to block concurrent updates at 1731341741488Preparing flush snapshotting stores in 8a30e5caa9f1f33bab5e05117c34d967 at 1731341741488Finished memstore snapshotting TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731341741488Flushing stores of TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967. at 1731341741489 (+1 ms)Flushing 8a30e5caa9f1f33bab5e05117c34d967/A: creating writer at 1731341741489Flushing 8a30e5caa9f1f33bab5e05117c34d967/A: appending metadata at 1731341741491 (+2 ms)Flushing 8a30e5caa9f1f33bab5e05117c34d967/A: closing flushed file at 1731341741492 (+1 ms)Flushing 8a30e5caa9f1f33bab5e05117c34d967/B: creating writer at 1731341741900 (+408 ms)Flushing 8a30e5caa9f1f33bab5e05117c34d967/B: appending metadata at 1731341741902 (+2 ms)Flushing 8a30e5caa9f1f33bab5e05117c34d967/B: closing flushed file at 1731341741902Flushing 8a30e5caa9f1f33bab5e05117c34d967/C: creating writer at 1731341742310 (+408 ms)Flushing 8a30e5caa9f1f33bab5e05117c34d967/C: appending metadata at 1731341742312 (+2 ms)Flushing 8a30e5caa9f1f33bab5e05117c34d967/C: closing flushed file at 1731341742312Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a2d8cfa: reopening flushed file at 1731341742720 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1543485b: reopening flushed file at 1731341742724 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d4854f0: reopening flushed file at 1731341742728 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8a30e5caa9f1f33bab5e05117c34d967 in 1243ms, sequenceid=974, compaction requested=true at 1731341742732 (+4 ms)Writing region close event to WAL at 1731341742842 (+110 ms)Running coprocessor post-close hooks at 1731341742845 (+3 ms)Closed at 1731341742845 2024-11-11T16:15:42,847 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:42,847 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=8a30e5caa9f1f33bab5e05117c34d967, regionState=CLOSED 2024-11-11T16:15:42,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:15:42,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T16:15:42,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a30e5caa9f1f33bab5e05117c34d967, server=16b413a53992,45121,1731341648557 in 1.5130 sec 2024-11-11T16:15:42,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-11T16:15:42,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a30e5caa9f1f33bab5e05117c34d967, UNASSIGN in 1.5160 sec 2024-11-11T16:15:42,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T16:15:42,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5190 sec 2024-11-11T16:15:42,854 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341742854"}]},"ts":"1731341742854"} 2024-11-11T16:15:42,855 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T16:15:42,855 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T16:15:42,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5280 sec 2024-11-11T16:15:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T16:15:43,486 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:43,486 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T16:15:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:43,488 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T16:15:43,488 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:43,490 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:43,492 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/recovered.edits] 2024-11-11T16:15:43,494 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d6b0d739102548be8da0a043837887e2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d6b0d739102548be8da0a043837887e2 2024-11-11T16:15:43,495 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d91685203ef7462b968643023f651658 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/d91685203ef7462b968643023f651658 2024-11-11T16:15:43,496 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/daaa29eeca064836b4947e1fea2cdf8b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/A/daaa29eeca064836b4947e1fea2cdf8b 2024-11-11T16:15:43,498 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4319af3b4ee541bfab275242e5af0759 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4319af3b4ee541bfab275242e5af0759 2024-11-11T16:15:43,499 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4d75f5a2406c48f09e867bed3acf65a1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/4d75f5a2406c48f09e867bed3acf65a1 2024-11-11T16:15:43,500 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e9bde28afecb467b9a223df9a59a52fa to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/B/e9bde28afecb467b9a223df9a59a52fa 2024-11-11T16:15:43,502 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/7ad3bebc7d9e42b3bad688e1ad158335 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/7ad3bebc7d9e42b3bad688e1ad158335 2024-11-11T16:15:43,503 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/de46993825a042b1ac8468a2e6aaa7ac to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/de46993825a042b1ac8468a2e6aaa7ac 2024-11-11T16:15:43,504 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e6719663009849c89dabab77ce2dc46f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/C/e6719663009849c89dabab77ce2dc46f 2024-11-11T16:15:43,506 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/recovered.edits/977.seqid to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967/recovered.edits/977.seqid 2024-11-11T16:15:43,506 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/8a30e5caa9f1f33bab5e05117c34d967 2024-11-11T16:15:43,506 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T16:15:43,508 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:43,512 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T16:15:43,514 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T16:15:43,515 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:43,515 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T16:15:43,515 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731341743515"}]},"ts":"9223372036854775807"} 2024-11-11T16:15:43,517 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T16:15:43,517 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8a30e5caa9f1f33bab5e05117c34d967, NAME => 'TestAcidGuarantees,,1731341712105.8a30e5caa9f1f33bab5e05117c34d967.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T16:15:43,517 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T16:15:43,517 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731341743517"}]},"ts":"9223372036854775807"} 2024-11-11T16:15:43,519 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T16:15:43,519 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:43,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-11T16:15:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T16:15:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T16:15:43,596 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:43,606 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=453 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=665 (was 696), ProcessCount=11 (was 11), AvailableMemoryMB=1420 (was 1267) - AvailableMemoryMB LEAK? - 2024-11-11T16:15:43,615 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=665, ProcessCount=11, AvailableMemoryMB=1420 2024-11-11T16:15:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:15:43,616 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:15:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:43,618 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T16:15:43,618 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:43,619 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T16:15:43,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-11T16:15:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T16:15:43,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742165_1341 (size=1061) 2024-11-11T16:15:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T16:15:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T16:15:44,026 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:15:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742166_1342 (size=53) 2024-11-11T16:15:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T16:15:44,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:44,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 79645af1f242c463cd05cc60d56c4bdc, disabling compactions & flushes 2024-11-11T16:15:44,435 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. after waiting 0 ms 2024-11-11T16:15:44,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,435 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,435 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 79645af1f242c463cd05cc60d56c4bdc: Waiting for close lock at 1731341744435Disabling compacts and flushes for region at 1731341744435Disabling writes for close at 1731341744435Writing region close event to WAL at 1731341744435Closed at 1731341744435 2024-11-11T16:15:44,436 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T16:15:44,437 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731341744436"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341744436"}]},"ts":"1731341744436"} 2024-11-11T16:15:44,438 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T16:15:44,439 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T16:15:44,439 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341744439"}]},"ts":"1731341744439"} 2024-11-11T16:15:44,440 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T16:15:44,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, ASSIGN}] 2024-11-11T16:15:44,441 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, ASSIGN 2024-11-11T16:15:44,442 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, ASSIGN; state=OFFLINE, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=false 2024-11-11T16:15:44,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:44,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, ASSIGN because future has completed 2024-11-11T16:15:44,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:15:44,749 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,749 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:15:44,750 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,750 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:44,750 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,750 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,751 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,752 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:44,752 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79645af1f242c463cd05cc60d56c4bdc columnFamilyName A 2024-11-11T16:15:44,753 DEBUG [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:44,753 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(327): Store=79645af1f242c463cd05cc60d56c4bdc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:44,753 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,754 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:44,754 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79645af1f242c463cd05cc60d56c4bdc columnFamilyName B 2024-11-11T16:15:44,754 DEBUG [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:44,754 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(327): Store=79645af1f242c463cd05cc60d56c4bdc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:44,754 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,755 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:44,755 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79645af1f242c463cd05cc60d56c4bdc columnFamilyName C 2024-11-11T16:15:44,755 DEBUG [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:44,756 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(327): Store=79645af1f242c463cd05cc60d56c4bdc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:44,756 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,756 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T16:15:44,756 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,757 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,758 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,758 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,758 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:15:44,759 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,761 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:15:44,761 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 79645af1f242c463cd05cc60d56c4bdc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70847469, jitterRate=0.05570955574512482}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:15:44,761 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:44,762 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 79645af1f242c463cd05cc60d56c4bdc: Running coprocessor pre-open hook at 1731341744750Writing region info on filesystem at 1731341744750Initializing all the Stores at 1731341744751 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341744751Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341744751Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341744751Cleaning up temporary data from old regions at 1731341744758 (+7 ms)Running coprocessor post-open hooks at 1731341744761 (+3 ms)Region opened successfully at 1731341744762 (+1 ms) 2024-11-11T16:15:44,762 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., pid=85, masterSystemTime=1731341744746 2024-11-11T16:15:44,764 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,764 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:44,764 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:44,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:15:44,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T16:15:44,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 in 171 msec 2024-11-11T16:15:44,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T16:15:44,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, ASSIGN in 326 msec 2024-11-11T16:15:44,769 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T16:15:44,769 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341744769"}]},"ts":"1731341744769"} 2024-11-11T16:15:44,771 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T16:15:44,771 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T16:15:44,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-11T16:15:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-11T16:15:45,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:45,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385f6522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:45,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:45,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:45,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb76b44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:45,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da35700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:45,781 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:45,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:45,783 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:15:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,33601,1731341647570 2024-11-11T16:15:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2fdde94a 2024-11-11T16:15:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:15:45,785 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:15:45,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:15:45,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:15:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T16:15:45,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742167_1343 (size=1097) 2024-11-11T16:15:46,197 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T16:15:46,197 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T16:15:46,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:15:46,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, REOPEN/MOVE}] 2024-11-11T16:15:46,202 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, REOPEN/MOVE 2024-11-11T16:15:46,202 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, REOPEN/MOVE because future has completed 2024-11-11T16:15:46,204 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:15:46,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:15:46,356 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,356 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:15:46,356 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 79645af1f242c463cd05cc60d56c4bdc, disabling compactions & flushes 2024-11-11T16:15:46,356 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,357 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,357 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. after waiting 0 ms 2024-11-11T16:15:46,357 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,363 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:15:46,364 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,364 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 79645af1f242c463cd05cc60d56c4bdc: Waiting for close lock at 1731341746356Running coprocessor pre-close hooks at 1731341746356Disabling compacts and flushes for region at 1731341746356Disabling writes for close at 1731341746357 (+1 ms)Writing region close event to WAL at 1731341746357Running coprocessor post-close hooks at 1731341746364 (+7 ms)Closed at 1731341746364 2024-11-11T16:15:46,364 WARN [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 79645af1f242c463cd05cc60d56c4bdc to self. 2024-11-11T16:15:46,366 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,367 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=CLOSED 2024-11-11T16:15:46,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:15:46,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T16:15:46,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 in 165 msec 2024-11-11T16:15:46,372 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, REOPEN/MOVE; state=CLOSED, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=true 2024-11-11T16:15:46,523 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, REOPEN/MOVE because future has completed 2024-11-11T16:15:46,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:15:46,679 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,679 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:15:46,679 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,680 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:15:46,680 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,680 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,681 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,682 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:46,682 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79645af1f242c463cd05cc60d56c4bdc columnFamilyName A 2024-11-11T16:15:46,683 DEBUG [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:46,683 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(327): Store=79645af1f242c463cd05cc60d56c4bdc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:46,684 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,684 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:46,684 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79645af1f242c463cd05cc60d56c4bdc columnFamilyName B 2024-11-11T16:15:46,684 DEBUG [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:46,685 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(327): Store=79645af1f242c463cd05cc60d56c4bdc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:46,685 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,685 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:15:46,685 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79645af1f242c463cd05cc60d56c4bdc columnFamilyName C 2024-11-11T16:15:46,686 DEBUG [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:46,686 INFO [StoreOpener-79645af1f242c463cd05cc60d56c4bdc-1 {}] regionserver.HStore(327): Store=79645af1f242c463cd05cc60d56c4bdc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:15:46,686 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,686 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,688 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,688 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,688 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,689 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:15:46,690 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,690 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 79645af1f242c463cd05cc60d56c4bdc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66678077, jitterRate=-0.00641922652721405}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:15:46,690 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,691 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 79645af1f242c463cd05cc60d56c4bdc: Running coprocessor pre-open hook at 1731341746680Writing region info on filesystem at 1731341746680Initializing all the Stores at 1731341746681 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341746681Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341746681Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341746681Cleaning up temporary data from old regions at 1731341746688 (+7 ms)Running coprocessor post-open hooks at 1731341746690 (+2 ms)Region opened successfully at 1731341746691 (+1 ms) 2024-11-11T16:15:46,692 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., pid=90, masterSystemTime=1731341746676 2024-11-11T16:15:46,693 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,693 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,694 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=OPEN, openSeqNum=5, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:15:46,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-11T16:15:46,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 in 172 msec 2024-11-11T16:15:46,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T16:15:46,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, REOPEN/MOVE in 496 msec 2024-11-11T16:15:46,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-11T16:15:46,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 501 msec 2024-11-11T16:15:46,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 915 msec 2024-11-11T16:15:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T16:15:46,705 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:46,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62aeaeee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,706 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f00c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,708 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516788c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@702456a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,710 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@628a4c3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,711 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72dda4b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4a3604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3e5700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,714 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7415ce52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ce2342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,716 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be19a2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,717 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a48c14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb3b663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,719 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672c80a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745f2f68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38468ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60053b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,723 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91d8cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@819e07a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,724 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b92d891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,725 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a5bc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140cd51c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,727 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0d6d00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,728 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a770b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d55336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,730 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a285181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,731 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63365645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ffe2d31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:15:46,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:15:46,732 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:15:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:15:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:15:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637f698, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:15:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:15:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:15:46,734 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:15:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae3c0d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:15:46,735 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,735 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,735 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,735 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,735 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,745 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,745 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,747 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-11T16:15:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,751 DEBUG [Thread-1568 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T16:15:46,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,752 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:46,752 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:46,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:46,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:46,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:46,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:46,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:46,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:46,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:46,759 DEBUG [Thread-1575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,759 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:15:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:15:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:15:46,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:15:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:15:46,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117b40560a3b440cd80d7a8d211511c3f_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341746753/Put/seqid=0 2024-11-11T16:15:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341806779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341806788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341806788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742168_1344 (size=17034) 2024-11-11T16:15:46,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341806790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341806790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T16:15:46,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341806895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341806896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341806896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:46,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:46,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:46,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341806906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:46,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341806907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T16:15:47,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341807116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341807116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341807116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341807116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341807126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:47,201 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117b40560a3b440cd80d7a8d211511c3f_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117b40560a3b440cd80d7a8d211511c3f_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:47,203 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/262054fb947946678a22bb6691ea7989, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:47,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/262054fb947946678a22bb6691ea7989 is 175, key is test_row_0/A:col10/1731341746753/Put/seqid=0 2024-11-11T16:15:47,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:47,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742169_1345 (size=48139) 2024-11-11T16:15:47,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:47,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,220 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/262054fb947946678a22bb6691ea7989 2024-11-11T16:15:47,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/f680cef66d8d494d953f3130efada563 is 50, key is test_row_0/B:col10/1731341746753/Put/seqid=0 2024-11-11T16:15:47,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742170_1346 (size=12001) 2024-11-11T16:15:47,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/f680cef66d8d494d953f3130efada563 2024-11-11T16:15:47,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/02ce31344caf4ccdb9a5a845ac9cb64b is 50, key is test_row_0/C:col10/1731341746753/Put/seqid=0 2024-11-11T16:15:47,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742171_1347 (size=12001) 2024-11-11T16:15:47,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/02ce31344caf4ccdb9a5a845ac9cb64b 2024-11-11T16:15:47,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/262054fb947946678a22bb6691ea7989 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/262054fb947946678a22bb6691ea7989 2024-11-11T16:15:47,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/262054fb947946678a22bb6691ea7989, entries=250, sequenceid=16, filesize=47.0 K 2024-11-11T16:15:47,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/f680cef66d8d494d953f3130efada563 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f680cef66d8d494d953f3130efada563 2024-11-11T16:15:47,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:47,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:47,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f680cef66d8d494d953f3130efada563, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T16:15:47,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/02ce31344caf4ccdb9a5a845ac9cb64b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02ce31344caf4ccdb9a5a845ac9cb64b 2024-11-11T16:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T16:15:47,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02ce31344caf4ccdb9a5a845ac9cb64b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T16:15:47,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 79645af1f242c463cd05cc60d56c4bdc in 626ms, sequenceid=16, compaction requested=false 2024-11-11T16:15:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:47,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341746786, max timestamp=1731341747432Num uniques 859; 2024-11-11T16:15:47,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:47,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341746786, max timestamp=1731341747432Num uniques 859; 2024-11-11T16:15:47,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:47,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341746786, max timestamp=1731341747432Num uniques 859; 2024-11-11T16:15:47,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:47,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-11T16:15:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:47,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111149e9a0e0ff654116a6bb80552360c9f0_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341746786/Put/seqid=0 2024-11-11T16:15:47,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742172_1348 (size=12154) 2024-11-11T16:15:47,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:47,480 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111149e9a0e0ff654116a6bb80552360c9f0_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149e9a0e0ff654116a6bb80552360c9f0_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:47,481 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/92376040c8384af3973937c102a55b63, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:47,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/92376040c8384af3973937c102a55b63 is 175, key is test_row_0/A:col10/1731341746786/Put/seqid=0 2024-11-11T16:15:47,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742173_1349 (size=30955) 2024-11-11T16:15:47,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341747471, max timestamp=1731341747492Num uniques 859; 2024-11-11T16:15:47,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:47,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341747471, max timestamp=1731341747492Num uniques 859; 2024-11-11T16:15:47,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:47,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341747471, max timestamp=1731341747492Num uniques 859; 2024-11-11T16:15:47,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:47,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/92376040c8384af3973937c102a55b63 2024-11-11T16:15:47,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/d5412f2bba1c4b51849aba55560eed1e is 50, key is test_row_0/B:col10/1731341746786/Put/seqid=0 2024-11-11T16:15:47,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:47,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:47,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742174_1350 (size=12001) 2024-11-11T16:15:47,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/d5412f2bba1c4b51849aba55560eed1e 2024-11-11T16:15:47,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/e81258f861604a119fdfd75b41769cca is 50, key is test_row_0/C:col10/1731341746786/Put/seqid=0 2024-11-11T16:15:47,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742175_1351 (size=12001) 2024-11-11T16:15:47,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731341747492, max timestamp=1731341747554Num uniques 1068; 2024-11-11T16:15:47,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731341747492, max timestamp=1731341747554Num uniques 1068; 2024-11-11T16:15:47,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-11T16:15:47,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731341747492, max timestamp=1731341747554Num uniques 1068; 2024-11-11T16:15:47,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-11T16:15:47,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-11T16:15:47,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341807593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341807595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341807596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341807597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341807608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731341747584, max timestamp=1731341747607Num uniques 1177; 2024-11-11T16:15:47,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-11T16:15:47,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731341747584, max timestamp=1731341747607Num uniques 1177; 2024-11-11T16:15:47,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-11T16:15:47,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731341747584, max timestamp=1731341747607Num uniques 1177; 2024-11-11T16:15:47,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-11T16:15:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:47,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731341747735, max timestamp=1731341747758Num uniques 1136; 2024-11-11T16:15:47,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-11T16:15:47,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731341747735, max timestamp=1731341747758Num uniques 1136; 2024-11-11T16:15:47,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-11T16:15:47,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731341747735, max timestamp=1731341747758Num uniques 1136; 2024-11-11T16:15:47,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-11T16:15:47,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731341747791, max timestamp=1731341747813Num uniques 1145; 2024-11-11T16:15:47,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-11T16:15:47,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731341747791, max timestamp=1731341747813Num uniques 1145; 2024-11-11T16:15:47,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-11T16:15:47,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731341747791, max timestamp=1731341747813Num uniques 1145; 2024-11-11T16:15:47,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-11T16:15:47,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:47,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T16:15:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341807913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341807915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341807915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/e81258f861604a119fdfd75b41769cca 2024-11-11T16:15:47,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/92376040c8384af3973937c102a55b63 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/92376040c8384af3973937c102a55b63 2024-11-11T16:15:47,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341807938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341807945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:47,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/92376040c8384af3973937c102a55b63, entries=150, sequenceid=37, filesize=30.2 K 2024-11-11T16:15:47,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/d5412f2bba1c4b51849aba55560eed1e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/d5412f2bba1c4b51849aba55560eed1e 2024-11-11T16:15:47,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/d5412f2bba1c4b51849aba55560eed1e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T16:15:47,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/e81258f861604a119fdfd75b41769cca as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e81258f861604a119fdfd75b41769cca 2024-11-11T16:15:47,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:47,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:47,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:47,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e81258f861604a119fdfd75b41769cca, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T16:15:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:47,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=161.02 KB/164880 for 79645af1f242c463cd05cc60d56c4bdc in 563ms, sequenceid=37, compaction requested=false 2024-11-11T16:15:47,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:48,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=167.72 KB heapSize=467.86 KB 2024-11-11T16:15:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:48,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341747850, max timestamp=1731341748060Num uniques 1204; 2024-11-11T16:15:48,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-11T16:15:48,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341747850, max timestamp=1731341748060Num uniques 1204; 2024-11-11T16:15:48,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-11T16:15:48,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731341747850, max timestamp=1731341748060Num uniques 1204; 2024-11-11T16:15:48,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-11T16:15:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:48,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ba6c0e51318c4509bd6102537703c134_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341748061/Put/seqid=0 2024-11-11T16:15:48,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742176_1352 (size=14794) 2024-11-11T16:15:48,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:48,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ba6c0e51318c4509bd6102537703c134_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ba6c0e51318c4509bd6102537703c134_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:48,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b202beb8563545e9a86c4118b03451af, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:48,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b202beb8563545e9a86c4118b03451af is 175, key is test_row_0/A:col10/1731341748061/Put/seqid=0 2024-11-11T16:15:48,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742177_1353 (size=39749) 2024-11-11T16:15:48,111 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b202beb8563545e9a86c4118b03451af 2024-11-11T16:15:48,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341748089, max timestamp=1731341748119Num uniques 859; 2024-11-11T16:15:48,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:15:48,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341748089, max timestamp=1731341748119Num uniques 859; 2024-11-11T16:15:48,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341748089, max timestamp=1731341748119Num uniques 859; 2024-11-11T16:15:48,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:48,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:15:48,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/c37ba6bd73544695bf63a80a9873b9e7 is 50, key is test_row_0/B:col10/1731341748061/Put/seqid=0 2024-11-11T16:15:48,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742178_1354 (size=12151) 2024-11-11T16:15:48,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/c37ba6bd73544695bf63a80a9873b9e7 2024-11-11T16:15:48,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/6b976bd80ee64024b433275d13eb6a4a is 50, key is test_row_0/C:col10/1731341748061/Put/seqid=0 2024-11-11T16:15:48,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731341748162, max timestamp=1731341748190Num uniques 1168; 2024-11-11T16:15:48,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-11-11T16:15:48,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731341748162, max timestamp=1731341748190Num uniques 1068; 2024-11-11T16:15:48,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-11T16:15:48,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341748185, max timestamp=1731341748190Num uniques 1068; 2024-11-11T16:15:48,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-11T16:15:48,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742179_1355 (size=12151) 2024-11-11T16:15:48,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/6b976bd80ee64024b433275d13eb6a4a 2024-11-11T16:15:48,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b202beb8563545e9a86c4118b03451af as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b202beb8563545e9a86c4118b03451af 2024-11-11T16:15:48,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b202beb8563545e9a86c4118b03451af, entries=200, sequenceid=145, filesize=38.8 K 2024-11-11T16:15:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/c37ba6bd73544695bf63a80a9873b9e7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/c37ba6bd73544695bf63a80a9873b9e7 2024-11-11T16:15:48,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/c37ba6bd73544695bf63a80a9873b9e7, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T16:15:48,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/6b976bd80ee64024b433275d13eb6a4a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6b976bd80ee64024b433275d13eb6a4a 2024-11-11T16:15:48,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6b976bd80ee64024b433275d13eb6a4a, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T16:15:48,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=81.95 KB/83912 for 79645af1f242c463cd05cc60d56c4bdc in 179ms, sequenceid=145, compaction requested=true 2024-11-11T16:15:48,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:48,241 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:48,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:48,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:48,242 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:48,242 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:48,242 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:15:48,243 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,243 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/262054fb947946678a22bb6691ea7989, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/92376040c8384af3973937c102a55b63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b202beb8563545e9a86c4118b03451af] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=116.1 K 2024-11-11T16:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:48,243 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:48,243 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/262054fb947946678a22bb6691ea7989, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/92376040c8384af3973937c102a55b63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b202beb8563545e9a86c4118b03451af] 2024-11-11T16:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:48,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=95.36 KB heapSize=263.17 KB 2024-11-11T16:15:48,244 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36153 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:48,244 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:15:48,244 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:48,244 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f680cef66d8d494d953f3130efada563, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/d5412f2bba1c4b51849aba55560eed1e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/c37ba6bd73544695bf63a80a9873b9e7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=35.3 K 2024-11-11T16:15:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:48,244 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262054fb947946678a22bb6691ea7989, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341746747 2024-11-11T16:15:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:48,244 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f680cef66d8d494d953f3130efada563, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341746747 2024-11-11T16:15:48,245 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92376040c8384af3973937c102a55b63, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731341746786 2024-11-11T16:15:48,245 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d5412f2bba1c4b51849aba55560eed1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731341746786 2024-11-11T16:15:48,245 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b202beb8563545e9a86c4118b03451af, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731341747850 2024-11-11T16:15:48,248 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c37ba6bd73544695bf63a80a9873b9e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731341747850 2024-11-11T16:15:48,262 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:48,263 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:48,264 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/33ffdbc6f1c142b98ab56fcf9dc1aad9 is 50, key is test_row_0/B:col10/1731341748061/Put/seqid=0 2024-11-11T16:15:48,269 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111110ecb67e465f4fba82e3f2b2daa369c5_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:48,272 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111110ecb67e465f4fba82e3f2b2daa369c5_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:48,272 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110ecb67e465f4fba82e3f2b2daa369c5_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:48,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111de2c3cc279324d819fff4612f7eb1235_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341748238/Put/seqid=0 2024-11-11T16:15:48,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742180_1356 (size=12254) 2024-11-11T16:15:48,295 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/33ffdbc6f1c142b98ab56fcf9dc1aad9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/33ffdbc6f1c142b98ab56fcf9dc1aad9 2024-11-11T16:15:48,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341808280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:48,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,304 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into 33ffdbc6f1c142b98ab56fcf9dc1aad9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:48,304 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:48,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341808283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,305 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=13, startTime=1731341748242; duration=0sec 2024-11-11T16:15:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341808284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341808286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341808289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,305 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:48,308 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:15:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,308 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:48,310 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36153 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:48,311 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:15:48,311 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,311 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02ce31344caf4ccdb9a5a845ac9cb64b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e81258f861604a119fdfd75b41769cca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6b976bd80ee64024b433275d13eb6a4a] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=35.3 K 2024-11-11T16:15:48,311 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ce31344caf4ccdb9a5a845ac9cb64b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341746747 2024-11-11T16:15:48,311 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e81258f861604a119fdfd75b41769cca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731341746786 2024-11-11T16:15:48,312 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b976bd80ee64024b433275d13eb6a4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731341747850 2024-11-11T16:15:48,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742181_1357 (size=12304) 2024-11-11T16:15:48,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:48,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742182_1358 (size=4469) 2024-11-11T16:15:48,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111de2c3cc279324d819fff4612f7eb1235_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111de2c3cc279324d819fff4612f7eb1235_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:48,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/91996c89cfd44cddb4b23a0cc81d3cbd, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:48,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/91996c89cfd44cddb4b23a0cc81d3cbd is 175, key is test_row_0/A:col10/1731341748238/Put/seqid=0 2024-11-11T16:15:48,359 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:48,359 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/fd77a52c83a3422f9b6a48e8e21b0d5e is 50, key is test_row_0/C:col10/1731341748061/Put/seqid=0 2024-11-11T16:15:48,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742183_1359 (size=31105) 2024-11-11T16:15:48,385 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T16:15:48,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/91996c89cfd44cddb4b23a0cc81d3cbd 2024-11-11T16:15:48,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742184_1360 (size=12254) 2024-11-11T16:15:48,413 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/fd77a52c83a3422f9b6a48e8e21b0d5e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fd77a52c83a3422f9b6a48e8e21b0d5e 2024-11-11T16:15:48,425 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into fd77a52c83a3422f9b6a48e8e21b0d5e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:48,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,426 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:48,426 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=13, startTime=1731341748243; duration=0sec 2024-11-11T16:15:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341808409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,426 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:48,426 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:15:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341808416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341808417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341808417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341808417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/43f0453b7c1f48f08135cba5426a7b39 is 50, key is test_row_0/B:col10/1731341748238/Put/seqid=0 2024-11-11T16:15:48,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742185_1361 (size=12151) 2024-11-11T16:15:48,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/43f0453b7c1f48f08135cba5426a7b39 2024-11-11T16:15:48,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:48,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:48,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/5c58f7a13a244633a996291ad3678833 is 50, key is test_row_0/C:col10/1731341748238/Put/seqid=0 2024-11-11T16:15:48,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742186_1362 (size=12151) 2024-11-11T16:15:48,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341808636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341808636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341808636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341808636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341808636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:48,751 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#307 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:48,751 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/339101bb433949219ea4d92d0f68b4cc is 175, key is test_row_0/A:col10/1731341748061/Put/seqid=0 2024-11-11T16:15:48,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742187_1363 (size=31208) 2024-11-11T16:15:48,760 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/339101bb433949219ea4d92d0f68b4cc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/339101bb433949219ea4d92d0f68b4cc 2024-11-11T16:15:48,766 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into 339101bb433949219ea4d92d0f68b4cc(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:48,766 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:48,766 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=13, startTime=1731341748241; duration=0sec 2024-11-11T16:15:48,766 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:48,766 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:15:48,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:48,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:48,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:48,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=30.51 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/5c58f7a13a244633a996291ad3678833 2024-11-11T16:15:48,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/91996c89cfd44cddb4b23a0cc81d3cbd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/91996c89cfd44cddb4b23a0cc81d3cbd 2024-11-11T16:15:48,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/91996c89cfd44cddb4b23a0cc81d3cbd, entries=150, sequenceid=193, filesize=30.4 K 2024-11-11T16:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T16:15:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/43f0453b7c1f48f08135cba5426a7b39 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/43f0453b7c1f48f08135cba5426a7b39 2024-11-11T16:15:48,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/43f0453b7c1f48f08135cba5426a7b39, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T16:15:48,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/5c58f7a13a244633a996291ad3678833 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/5c58f7a13a244633a996291ad3678833 2024-11-11T16:15:48,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/5c58f7a13a244633a996291ad3678833, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T16:15:48,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.07 KB/104522, heapSize ~283.05 KB/289840, currentSize=80.51 KB/82440 for 79645af1f242c463cd05cc60d56c4bdc in 675ms, sequenceid=193, compaction requested=false 2024-11-11T16:15:48,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:48,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T16:15:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:48,921 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:15:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116cfc4ac249964852a91dfef0e92f1375_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341748285/Put/seqid=0 2024-11-11T16:15:48,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742188_1364 (size=12304) 2024-11-11T16:15:48,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:48,949 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116cfc4ac249964852a91dfef0e92f1375_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116cfc4ac249964852a91dfef0e92f1375_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:48,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a31ca8961a9540c3a83536a54dee9c2f, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a31ca8961a9540c3a83536a54dee9c2f is 175, key is test_row_0/A:col10/1731341748285/Put/seqid=0 2024-11-11T16:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:48,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:48,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742189_1365 (size=31105) 2024-11-11T16:15:48,968 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a31ca8961a9540c3a83536a54dee9c2f 2024-11-11T16:15:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/3201b186993c46d7bdf61e8c1506f212 is 50, key is test_row_0/B:col10/1731341748285/Put/seqid=0 2024-11-11T16:15:48,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742190_1366 (size=12151) 2024-11-11T16:15:49,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341808994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341808994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341808995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341808995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341808996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341809116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341809116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341809116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341809116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341809125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341809336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341809336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341809336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341809336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341809336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,386 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/3201b186993c46d7bdf61e8c1506f212 2024-11-11T16:15:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/0cff77898cda4c82b5787da2117ce00c is 50, key is test_row_0/C:col10/1731341748285/Put/seqid=0 2024-11-11T16:15:49,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742191_1367 (size=12151) 2024-11-11T16:15:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341809646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341809645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341809646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341809646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341809646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:49,809 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/0cff77898cda4c82b5787da2117ce00c 2024-11-11T16:15:49,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a31ca8961a9540c3a83536a54dee9c2f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a31ca8961a9540c3a83536a54dee9c2f 2024-11-11T16:15:49,821 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a31ca8961a9540c3a83536a54dee9c2f, entries=150, sequenceid=211, filesize=30.4 K 2024-11-11T16:15:49,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/3201b186993c46d7bdf61e8c1506f212 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3201b186993c46d7bdf61e8c1506f212 2024-11-11T16:15:49,825 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3201b186993c46d7bdf61e8c1506f212, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T16:15:49,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/0cff77898cda4c82b5787da2117ce00c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0cff77898cda4c82b5787da2117ce00c 2024-11-11T16:15:49,830 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0cff77898cda4c82b5787da2117ce00c, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T16:15:49,831 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 79645af1f242c463cd05cc60d56c4bdc in 910ms, sequenceid=211, compaction requested=true 2024-11-11T16:15:49,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:49,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:49,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-11T16:15:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-11T16:15:49,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T16:15:49,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0810 sec 2024-11-11T16:15:49,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 3.0870 sec 2024-11-11T16:15:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:50,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:15:50,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:50,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:50,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:50,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:50,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:50,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:50,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111245173e4eec2406bb5bd57b77c85e514_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341750168/Put/seqid=0 2024-11-11T16:15:50,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742192_1368 (size=14794) 2024-11-11T16:15:50,192 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:50,196 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111245173e4eec2406bb5bd57b77c85e514_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111245173e4eec2406bb5bd57b77c85e514_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:50,197 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a2d62d345b4444188818de958b3af689, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:50,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a2d62d345b4444188818de958b3af689 is 175, key is test_row_0/A:col10/1731341750168/Put/seqid=0 2024-11-11T16:15:50,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742193_1369 (size=39749) 2024-11-11T16:15:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341810190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341810191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341810192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341810193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341810195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341810306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341810316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341810316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341810316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341810316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341810526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341810536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341810536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341810536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341810536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,601 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a2d62d345b4444188818de958b3af689 2024-11-11T16:15:50,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/9077c4c321e84046b236d208243bc56d is 50, key is test_row_0/B:col10/1731341750168/Put/seqid=0 2024-11-11T16:15:50,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742194_1370 (size=12151) 2024-11-11T16:15:50,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341810846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341810846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341810846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341810856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:50,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341810856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:50,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T16:15:50,916 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:50,916 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-11T16:15:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T16:15:50,918 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:50,919 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:50,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:51,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/9077c4c321e84046b236d208243bc56d 2024-11-11T16:15:51,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/917c98cd3f664b5bbd2a010689234991 is 50, key is test_row_0/C:col10/1731341750168/Put/seqid=0 2024-11-11T16:15:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T16:15:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742195_1371 (size=12151) 2024-11-11T16:15:51,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/917c98cd3f664b5bbd2a010689234991 2024-11-11T16:15:51,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/a2d62d345b4444188818de958b3af689 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a2d62d345b4444188818de958b3af689 2024-11-11T16:15:51,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a2d62d345b4444188818de958b3af689, entries=200, sequenceid=230, filesize=38.8 K 2024-11-11T16:15:51,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/9077c4c321e84046b236d208243bc56d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9077c4c321e84046b236d208243bc56d 2024-11-11T16:15:51,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9077c4c321e84046b236d208243bc56d, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T16:15:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/917c98cd3f664b5bbd2a010689234991 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/917c98cd3f664b5bbd2a010689234991 2024-11-11T16:15:51,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/917c98cd3f664b5bbd2a010689234991, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T16:15:51,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 79645af1f242c463cd05cc60d56c4bdc in 892ms, sequenceid=230, compaction requested=true 2024-11-11T16:15:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:51,062 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:51,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:51,062 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:51,065 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48707 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:51,065 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:15:51,065 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,065 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/33ffdbc6f1c142b98ab56fcf9dc1aad9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/43f0453b7c1f48f08135cba5426a7b39, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3201b186993c46d7bdf61e8c1506f212, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9077c4c321e84046b236d208243bc56d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=47.6 K 2024-11-11T16:15:51,066 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133167 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:51,066 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:15:51,066 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,066 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ffdbc6f1c142b98ab56fcf9dc1aad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731341747850 2024-11-11T16:15:51,066 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/339101bb433949219ea4d92d0f68b4cc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/91996c89cfd44cddb4b23a0cc81d3cbd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a31ca8961a9540c3a83536a54dee9c2f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a2d62d345b4444188818de958b3af689] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=130.0 K 2024-11-11T16:15:51,066 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,066 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/339101bb433949219ea4d92d0f68b4cc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/91996c89cfd44cddb4b23a0cc81d3cbd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a31ca8961a9540c3a83536a54dee9c2f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a2d62d345b4444188818de958b3af689] 2024-11-11T16:15:51,066 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f0453b7c1f48f08135cba5426a7b39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731341748231 2024-11-11T16:15:51,066 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339101bb433949219ea4d92d0f68b4cc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731341747850 2024-11-11T16:15:51,066 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3201b186993c46d7bdf61e8c1506f212, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731341748280 2024-11-11T16:15:51,067 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91996c89cfd44cddb4b23a0cc81d3cbd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731341748231 2024-11-11T16:15:51,067 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9077c4c321e84046b236d208243bc56d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731341748991 2024-11-11T16:15:51,068 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a31ca8961a9540c3a83536a54dee9c2f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731341748280 2024-11-11T16:15:51,069 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d62d345b4444188818de958b3af689, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731341748991 2024-11-11T16:15:51,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T16:15:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,072 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:15:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:51,085 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:51,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111106abd73f4896429abd2e292f54b748a7_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341750187/Put/seqid=0 2024-11-11T16:15:51,094 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:51,094 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111356ed26d2b354aad972cce6764df98b1_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:51,095 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/4a10e0ceef2c4b468d2bb92ac7f2be63 is 50, key is test_row_0/B:col10/1731341750168/Put/seqid=0 2024-11-11T16:15:51,097 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111356ed26d2b354aad972cce6764df98b1_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:51,098 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111356ed26d2b354aad972cce6764df98b1_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:51,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742196_1372 (size=12304) 2024-11-11T16:15:51,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,113 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111106abd73f4896429abd2e292f54b748a7_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106abd73f4896429abd2e292f54b748a7_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:51,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2ef2ac2cb02a4bccb62c5f5bcdefa101, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2ef2ac2cb02a4bccb62c5f5bcdefa101 is 175, key is test_row_0/A:col10/1731341750187/Put/seqid=0 2024-11-11T16:15:51,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742197_1373 (size=12391) 2024-11-11T16:15:51,127 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/4a10e0ceef2c4b468d2bb92ac7f2be63 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/4a10e0ceef2c4b468d2bb92ac7f2be63 2024-11-11T16:15:51,134 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into 4a10e0ceef2c4b468d2bb92ac7f2be63(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:51,134 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:51,134 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=12, startTime=1731341751062; duration=0sec 2024-11-11T16:15:51,134 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:51,134 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:15:51,134 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:51,137 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48707 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:51,137 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:15:51,137 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,138 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fd77a52c83a3422f9b6a48e8e21b0d5e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/5c58f7a13a244633a996291ad3678833, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0cff77898cda4c82b5787da2117ce00c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/917c98cd3f664b5bbd2a010689234991] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=47.6 K 2024-11-11T16:15:51,138 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting fd77a52c83a3422f9b6a48e8e21b0d5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731341747850 2024-11-11T16:15:51,138 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c58f7a13a244633a996291ad3678833, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731341748231 2024-11-11T16:15:51,139 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cff77898cda4c82b5787da2117ce00c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731341748280 2024-11-11T16:15:51,140 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 917c98cd3f664b5bbd2a010689234991, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731341748991 2024-11-11T16:15:51,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742198_1374 (size=4469) 2024-11-11T16:15:51,144 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#318 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:51,145 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/fa5babdf25284d59b473499a75f849ee is 175, key is test_row_0/A:col10/1731341750168/Put/seqid=0 2024-11-11T16:15:51,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742199_1375 (size=31105) 2024-11-11T16:15:51,150 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2ef2ac2cb02a4bccb62c5f5bcdefa101 2024-11-11T16:15:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/29176ecba3414a04911b54d9b39e08b0 is 50, key is test_row_0/B:col10/1731341750187/Put/seqid=0 2024-11-11T16:15:51,177 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#322 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:51,178 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/cc315f99252a47eb9896158d94b1647d is 50, key is test_row_0/C:col10/1731341750168/Put/seqid=0 2024-11-11T16:15:51,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742200_1376 (size=31345) 2024-11-11T16:15:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742202_1378 (size=12391) 2024-11-11T16:15:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742201_1377 (size=12151) 2024-11-11T16:15:51,203 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/29176ecba3414a04911b54d9b39e08b0 2024-11-11T16:15:51,219 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/cc315f99252a47eb9896158d94b1647d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/cc315f99252a47eb9896158d94b1647d 2024-11-11T16:15:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c76f7cc61e8e44e7b183d887f4a5174e is 50, key is test_row_0/C:col10/1731341750187/Put/seqid=0 2024-11-11T16:15:51,232 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into cc315f99252a47eb9896158d94b1647d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:51,232 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:51,232 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=12, startTime=1731341751062; duration=0sec 2024-11-11T16:15:51,233 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:51,233 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T16:15:51,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742203_1379 (size=12151) 2024-11-11T16:15:51,250 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c76f7cc61e8e44e7b183d887f4a5174e 2024-11-11T16:15:51,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2ef2ac2cb02a4bccb62c5f5bcdefa101 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2ef2ac2cb02a4bccb62c5f5bcdefa101 2024-11-11T16:15:51,261 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2ef2ac2cb02a4bccb62c5f5bcdefa101, entries=150, sequenceid=243, filesize=30.4 K 2024-11-11T16:15:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/29176ecba3414a04911b54d9b39e08b0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/29176ecba3414a04911b54d9b39e08b0 2024-11-11T16:15:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,269 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/29176ecba3414a04911b54d9b39e08b0, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T16:15:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c76f7cc61e8e44e7b183d887f4a5174e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c76f7cc61e8e44e7b183d887f4a5174e 2024-11-11T16:15:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,276 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c76f7cc61e8e44e7b183d887f4a5174e, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T16:15:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,277 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 79645af1f242c463cd05cc60d56c4bdc in 206ms, sequenceid=243, compaction requested=false 2024-11-11T16:15:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-11T16:15:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-11T16:15:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-11T16:15:51,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 362 msec 2024-11-11T16:15:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 368 msec 2024-11-11T16:15:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T16:15:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,546 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,547 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T16:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T16:15:51,550 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,551 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:51,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c42cb05591bf43f2ac4aec986e2521e1_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341751445/Put/seqid=0 2024-11-11T16:15:51,585 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/fa5babdf25284d59b473499a75f849ee as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fa5babdf25284d59b473499a75f849ee 2024-11-11T16:15:51,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742204_1380 (size=22268) 2024-11-11T16:15:51,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:51,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c42cb05591bf43f2ac4aec986e2521e1_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c42cb05591bf43f2ac4aec986e2521e1_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:51,598 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/163a1f486d7a4b178e7b8a995699447a, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:51,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/163a1f486d7a4b178e7b8a995699447a is 175, key is test_row_0/A:col10/1731341751445/Put/seqid=0 2024-11-11T16:15:51,606 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into fa5babdf25284d59b473499a75f849ee(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:51,606 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:51,606 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=12, startTime=1731341751062; duration=0sec 2024-11-11T16:15:51,606 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:51,606 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:15:51,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742205_1381 (size=65673) 2024-11-11T16:15:51,609 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/163a1f486d7a4b178e7b8a995699447a 2024-11-11T16:15:51,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/70ce831d62e4442e98bb315fedb954e7 is 50, key is test_row_0/B:col10/1731341751445/Put/seqid=0 2024-11-11T16:15:51,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742206_1382 (size=12151) 2024-11-11T16:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T16:15:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341811645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341751639, max timestamp=1731341751658Num uniques 859; 2024-11-11T16:15:51,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341751639, max timestamp=1731341751658Num uniques 859; 2024-11-11T16:15:51,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:51,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:51,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341751639, max timestamp=1731341751658Num uniques 859; 2024-11-11T16:15:51,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T16:15:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:51,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341811708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341811710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341811712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341811712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341811765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341811816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341811836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341811836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341811836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:51,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T16:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:51,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T16:15:51,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341811985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T16:15:52,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:52,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/70ce831d62e4442e98bb315fedb954e7 2024-11-11T16:15:52,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/1547ed65dd404f4490e77f8e353c6f60 is 50, key is test_row_0/C:col10/1731341751445/Put/seqid=0 2024-11-11T16:15:52,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742207_1383 (size=12151) 2024-11-11T16:15:52,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341812036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341812046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341812047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341812056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T16:15:52,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:52,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T16:15:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341812296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T16:15:52,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:52,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341812356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341812366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341812366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341812375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/1547ed65dd404f4490e77f8e353c6f60 2024-11-11T16:15:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/163a1f486d7a4b178e7b8a995699447a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/163a1f486d7a4b178e7b8a995699447a 2024-11-11T16:15:52,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/163a1f486d7a4b178e7b8a995699447a, entries=350, sequenceid=255, filesize=64.1 K 2024-11-11T16:15:52,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/70ce831d62e4442e98bb315fedb954e7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/70ce831d62e4442e98bb315fedb954e7 2024-11-11T16:15:52,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/70ce831d62e4442e98bb315fedb954e7, entries=150, sequenceid=255, filesize=11.9 K 2024-11-11T16:15:52,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/1547ed65dd404f4490e77f8e353c6f60 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1547ed65dd404f4490e77f8e353c6f60 2024-11-11T16:15:52,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1547ed65dd404f4490e77f8e353c6f60, entries=150, sequenceid=255, filesize=11.9 K 2024-11-11T16:15:52,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 79645af1f242c463cd05cc60d56c4bdc in 913ms, sequenceid=255, compaction requested=true 2024-11-11T16:15:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:52,467 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:52,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:52,467 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:52,468 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128123 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:52,468 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:15:52,468 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,469 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fa5babdf25284d59b473499a75f849ee, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2ef2ac2cb02a4bccb62c5f5bcdefa101, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/163a1f486d7a4b178e7b8a995699447a] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=125.1 K 2024-11-11T16:15:52,469 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,469 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fa5babdf25284d59b473499a75f849ee, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2ef2ac2cb02a4bccb62c5f5bcdefa101, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/163a1f486d7a4b178e7b8a995699447a] 2024-11-11T16:15:52,469 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:52,469 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:15:52,469 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,469 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5babdf25284d59b473499a75f849ee, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731341748991 2024-11-11T16:15:52,469 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/4a10e0ceef2c4b468d2bb92ac7f2be63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/29176ecba3414a04911b54d9b39e08b0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/70ce831d62e4442e98bb315fedb954e7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=35.8 K 2024-11-11T16:15:52,470 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a10e0ceef2c4b468d2bb92ac7f2be63, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731341748991 2024-11-11T16:15:52,470 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ef2ac2cb02a4bccb62c5f5bcdefa101, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731341750187 2024-11-11T16:15:52,470 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 29176ecba3414a04911b54d9b39e08b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731341750187 2024-11-11T16:15:52,470 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163a1f486d7a4b178e7b8a995699447a, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731341751438 2024-11-11T16:15:52,470 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ce831d62e4442e98bb315fedb954e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731341751445 2024-11-11T16:15:52,476 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:52,478 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:52,479 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/e7346c0b53db4b3d9ce2c1d10acbc7ba is 50, key is test_row_0/B:col10/1731341751445/Put/seqid=0 2024-11-11T16:15:52,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T16:15:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,479 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-11T16:15:52,480 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411113d3b50f2cf694852aa171fb91fab08f6_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:52,482 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411113d3b50f2cf694852aa171fb91fab08f6_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:52,482 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113d3b50f2cf694852aa171fb91fab08f6_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742208_1384 (size=12493) 2024-11-11T16:15:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111135183f5cdbaa42ddbed897963d6956d6_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341751708/Put/seqid=0 2024-11-11T16:15:52,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742209_1385 (size=4469) 2024-11-11T16:15:52,500 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#327 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:52,501 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/d73843cee6a24917a4614a3ae9012ca1 is 175, key is test_row_0/A:col10/1731341751445/Put/seqid=0 2024-11-11T16:15:52,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742210_1386 (size=12454) 2024-11-11T16:15:52,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742211_1387 (size=31447) 2024-11-11T16:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T16:15:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:52,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:52,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341812882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,895 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/e7346c0b53db4b3d9ce2c1d10acbc7ba as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/e7346c0b53db4b3d9ce2c1d10acbc7ba 2024-11-11T16:15:52,902 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into e7346c0b53db4b3d9ce2c1d10acbc7ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:52,902 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:52,902 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=13, startTime=1731341752467; duration=0sec 2024-11-11T16:15:52,902 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:52,902 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:15:52,903 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:52,903 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:52,904 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:15:52,904 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:52,904 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/cc315f99252a47eb9896158d94b1647d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c76f7cc61e8e44e7b183d887f4a5174e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1547ed65dd404f4490e77f8e353c6f60] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=35.8 K 2024-11-11T16:15:52,904 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting cc315f99252a47eb9896158d94b1647d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731341748991 2024-11-11T16:15:52,904 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c76f7cc61e8e44e7b183d887f4a5174e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731341750187 2024-11-11T16:15:52,905 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1547ed65dd404f4490e77f8e353c6f60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731341751445 2024-11-11T16:15:52,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341812886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341812886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341812886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341812895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,912 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111135183f5cdbaa42ddbed897963d6956d6_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111135183f5cdbaa42ddbed897963d6956d6_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/78aa56763bd34387bb242c96dff16e95, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:52,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/78aa56763bd34387bb242c96dff16e95 is 175, key is test_row_0/A:col10/1731341751708/Put/seqid=0 2024-11-11T16:15:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:52,918 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/d73843cee6a24917a4614a3ae9012ca1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/d73843cee6a24917a4614a3ae9012ca1 2024-11-11T16:15:52,923 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into d73843cee6a24917a4614a3ae9012ca1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:52,923 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:52,923 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=13, startTime=1731341752467; duration=0sec 2024-11-11T16:15:52,923 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:52,923 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:15:52,926 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:52,926 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/346522399fdf4543a36aa880ea4600b9 is 50, key is test_row_0/C:col10/1731341751445/Put/seqid=0 2024-11-11T16:15:52,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742212_1388 (size=31255) 2024-11-11T16:15:52,951 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/78aa56763bd34387bb242c96dff16e95 2024-11-11T16:15:52,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742213_1389 (size=12493) 2024-11-11T16:15:52,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/25dc7b494c5748069d3c2a3a9ae1bb74 is 50, key is test_row_0/B:col10/1731341751708/Put/seqid=0 2024-11-11T16:15:52,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742214_1390 (size=12301) 2024-11-11T16:15:52,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341812995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341813016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341813206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341813236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,363 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/346522399fdf4543a36aa880ea4600b9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/346522399fdf4543a36aa880ea4600b9 2024-11-11T16:15:53,368 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into 346522399fdf4543a36aa880ea4600b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:53,368 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:53,369 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=13, startTime=1731341752467; duration=0sec 2024-11-11T16:15:53,369 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:53,369 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:15:53,385 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/25dc7b494c5748069d3c2a3a9ae1bb74 2024-11-11T16:15:53,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/aff5293843fb4c9db78e78dc9f51b5e9 is 50, key is test_row_0/C:col10/1731341751708/Put/seqid=0 2024-11-11T16:15:53,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742215_1391 (size=12301) 2024-11-11T16:15:53,406 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/aff5293843fb4c9db78e78dc9f51b5e9 2024-11-11T16:15:53,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/78aa56763bd34387bb242c96dff16e95 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/78aa56763bd34387bb242c96dff16e95 2024-11-11T16:15:53,418 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/78aa56763bd34387bb242c96dff16e95, entries=150, sequenceid=292, filesize=30.5 K 2024-11-11T16:15:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/25dc7b494c5748069d3c2a3a9ae1bb74 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/25dc7b494c5748069d3c2a3a9ae1bb74 2024-11-11T16:15:53,426 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/25dc7b494c5748069d3c2a3a9ae1bb74, entries=150, sequenceid=292, filesize=12.0 K 2024-11-11T16:15:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/aff5293843fb4c9db78e78dc9f51b5e9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/aff5293843fb4c9db78e78dc9f51b5e9 2024-11-11T16:15:53,434 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/aff5293843fb4c9db78e78dc9f51b5e9, entries=150, sequenceid=292, filesize=12.0 K 2024-11-11T16:15:53,436 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 79645af1f242c463cd05cc60d56c4bdc in 957ms, sequenceid=292, compaction requested=false 2024-11-11T16:15:53,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:53,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:53,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T16:15:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T16:15:53,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T16:15:53,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8870 sec 2024-11-11T16:15:53,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8930 sec 2024-11-11T16:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:53,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:53,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:53,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112c4a675690fd4205aadb9015f5646885_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341752884/Put/seqid=0 2024-11-11T16:15:53,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742216_1392 (size=14994) 2024-11-11T16:15:53,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341753610, max timestamp=1731341753628Num uniques 859; 2024-11-11T16:15:53,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341753610, max timestamp=1731341753628Num uniques 859; 2024-11-11T16:15:53,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341753610, max timestamp=1731341753628Num uniques 859; 2024-11-11T16:15:53,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:53,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:53,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T16:15:53,706 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:53,707 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T16:15:53,709 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T16:15:53,710 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:53,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:53,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341813756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341813755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T16:15:53,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:53,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:53,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:53,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341813876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341813876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341813915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,934 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:53,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341813925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,935 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341813936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:53,951 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:53,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:53,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112c4a675690fd4205aadb9015f5646885_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112c4a675690fd4205aadb9015f5646885_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:53,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/4979fa82b3a14b87b3e76645d9d609f4, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:53,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/4979fa82b3a14b87b3e76645d9d609f4 is 175, key is test_row_0/A:col10/1731341752884/Put/seqid=0 2024-11-11T16:15:53,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742217_1393 (size=39949) 2024-11-11T16:15:54,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T16:15:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341814085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341814086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:54,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:54,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:54,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T16:15:54,368 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/4979fa82b3a14b87b3e76645d9d609f4 2024-11-11T16:15:54,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/82425fc9517747fe8a052dccdc48d133 is 50, key is test_row_0/B:col10/1731341752884/Put/seqid=0 2024-11-11T16:15:54,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341814406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:54,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341814409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:54,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742218_1394 (size=12301) 2024-11-11T16:15:54,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:54,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:54,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:54,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:54,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:54,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:54,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/82425fc9517747fe8a052dccdc48d133 2024-11-11T16:15:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T16:15:54,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/fee582438eb142e0bc4ac2359e6e9cf3 is 50, key is test_row_0/C:col10/1731341752884/Put/seqid=0 2024-11-11T16:15:54,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742219_1395 (size=12301) 2024-11-11T16:15:54,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/fee582438eb142e0bc4ac2359e6e9cf3 2024-11-11T16:15:54,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/4979fa82b3a14b87b3e76645d9d609f4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/4979fa82b3a14b87b3e76645d9d609f4 2024-11-11T16:15:54,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/4979fa82b3a14b87b3e76645d9d609f4, entries=200, sequenceid=305, filesize=39.0 K 2024-11-11T16:15:54,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/82425fc9517747fe8a052dccdc48d133 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/82425fc9517747fe8a052dccdc48d133 2024-11-11T16:15:54,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/82425fc9517747fe8a052dccdc48d133, entries=150, sequenceid=305, filesize=12.0 K 2024-11-11T16:15:54,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/fee582438eb142e0bc4ac2359e6e9cf3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fee582438eb142e0bc4ac2359e6e9cf3 2024-11-11T16:15:54,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fee582438eb142e0bc4ac2359e6e9cf3, entries=150, sequenceid=305, filesize=12.0 K 2024-11-11T16:15:54,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 79645af1f242c463cd05cc60d56c4bdc in 1398ms, sequenceid=305, compaction requested=true 2024-11-11T16:15:54,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:54,933 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:54,934 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:54,934 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:15:54,934 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,934 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/d73843cee6a24917a4614a3ae9012ca1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/78aa56763bd34387bb242c96dff16e95, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/4979fa82b3a14b87b3e76645d9d609f4] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=100.2 K 2024-11-11T16:15:54,934 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,934 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/d73843cee6a24917a4614a3ae9012ca1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/78aa56763bd34387bb242c96dff16e95, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/4979fa82b3a14b87b3e76645d9d609f4] 2024-11-11T16:15:54,934 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d73843cee6a24917a4614a3ae9012ca1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731341751445 2024-11-11T16:15:54,935 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78aa56763bd34387bb242c96dff16e95, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731341751678 2024-11-11T16:15:54,935 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4979fa82b3a14b87b3e76645d9d609f4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731341752869 2024-11-11T16:15:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:54,937 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:54,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:54,938 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:54,938 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:15:54,938 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,938 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/e7346c0b53db4b3d9ce2c1d10acbc7ba, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/25dc7b494c5748069d3c2a3a9ae1bb74, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/82425fc9517747fe8a052dccdc48d133] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=36.2 K 2024-11-11T16:15:54,939 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e7346c0b53db4b3d9ce2c1d10acbc7ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731341751445 2024-11-11T16:15:54,939 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 25dc7b494c5748069d3c2a3a9ae1bb74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731341751678 2024-11-11T16:15:54,940 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 82425fc9517747fe8a052dccdc48d133, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731341752869 2024-11-11T16:15:54,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341753687, max timestamp=1731341754947Num uniques 1018; 2024-11-11T16:15:54,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T16:15:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:54,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-11-11T16:15:54,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:54,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341753687, max timestamp=1731341754947Num uniques 1018; 2024-11-11T16:15:54,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T16:15:54,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:54,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:54,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:54,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:54,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T16:15:54,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:54,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,957 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:54,983 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:54,984 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/2ba0229ae1ca44aeb54037c0ffef15eb is 50, key is test_row_0/B:col10/1731341752884/Put/seqid=0 2024-11-11T16:15:54,996 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411113fe91bcda1864853bf3453b3007c7e73_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:54,998 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411113fe91bcda1864853bf3453b3007c7e73_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:54,999 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113fe91bcda1864853bf3453b3007c7e73_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:55,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113535aeac14a84ad0b4b23fe0fdbb99b3_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341753687/Put/seqid=0 2024-11-11T16:15:55,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742220_1396 (size=12745) 2024-11-11T16:15:55,033 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/2ba0229ae1ca44aeb54037c0ffef15eb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/2ba0229ae1ca44aeb54037c0ffef15eb 2024-11-11T16:15:55,040 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into 2ba0229ae1ca44aeb54037c0ffef15eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:55,040 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:55,040 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=13, startTime=1731341754937; duration=0sec 2024-11-11T16:15:55,041 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:55,041 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:15:55,041 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:15:55,042 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:15:55,042 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:15:55,042 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,042 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/346522399fdf4543a36aa880ea4600b9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/aff5293843fb4c9db78e78dc9f51b5e9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fee582438eb142e0bc4ac2359e6e9cf3] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=36.2 K 2024-11-11T16:15:55,042 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 346522399fdf4543a36aa880ea4600b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731341751445 2024-11-11T16:15:55,042 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting aff5293843fb4c9db78e78dc9f51b5e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731341751678 2024-11-11T16:15:55,043 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting fee582438eb142e0bc4ac2359e6e9cf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731341752869 2024-11-11T16:15:55,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742221_1397 (size=4469) 2024-11-11T16:15:55,057 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:55,058 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/d11cd12fff66432aae13d4ed32fd9d9e is 50, key is test_row_0/C:col10/1731341752884/Put/seqid=0 2024-11-11T16:15:55,060 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#336 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:55,061 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/c1d074b164124e97be51b5dd3fd9e2eb is 175, key is test_row_0/A:col10/1731341752884/Put/seqid=0 2024-11-11T16:15:55,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742222_1398 (size=14994) 2024-11-11T16:15:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742223_1399 (size=12745) 2024-11-11T16:15:55,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742224_1400 (size=31699) 2024-11-11T16:15:55,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341815106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341815109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341815237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341815238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,464 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:55,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341815455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341815455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,468 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113535aeac14a84ad0b4b23fe0fdbb99b3_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113535aeac14a84ad0b4b23fe0fdbb99b3_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:55,469 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b2dd3269dc454d3db58ad80d9e2f5103, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:55,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b2dd3269dc454d3db58ad80d9e2f5103 is 175, key is test_row_0/A:col10/1731341753687/Put/seqid=0 2024-11-11T16:15:55,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742225_1401 (size=39949) 2024-11-11T16:15:55,512 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/d11cd12fff66432aae13d4ed32fd9d9e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/d11cd12fff66432aae13d4ed32fd9d9e 2024-11-11T16:15:55,518 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/c1d074b164124e97be51b5dd3fd9e2eb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/c1d074b164124e97be51b5dd3fd9e2eb 2024-11-11T16:15:55,518 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into d11cd12fff66432aae13d4ed32fd9d9e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:55,518 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:55,518 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=13, startTime=1731341754937; duration=0sec 2024-11-11T16:15:55,519 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:55,519 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:15:55,523 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into c1d074b164124e97be51b5dd3fd9e2eb(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:55,523 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:55,523 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=13, startTime=1731341754932; duration=0sec 2024-11-11T16:15:55,523 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:55,523 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:15:55,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341815776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341815781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T16:15:55,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b2dd3269dc454d3db58ad80d9e2f5103 2024-11-11T16:15:55,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:55,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:55,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:55,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:55,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/8b6827f814224299a29ef63f347ad22b is 50, key is test_row_0/B:col10/1731341753687/Put/seqid=0 2024-11-11T16:15:55,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742226_1402 (size=12301) 2024-11-11T16:15:55,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/8b6827f814224299a29ef63f347ad22b 2024-11-11T16:15:55,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341815955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341815956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,962 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,963 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:55,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/1cae9f6d99984f2fb8cea0dfdcfdf631 is 50, key is test_row_0/C:col10/1731341753687/Put/seqid=0 2024-11-11T16:15:55,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341815968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:55,980 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:15:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:56,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742227_1403 (size=12301) 2024-11-11T16:15:56,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:56,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:56,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:56,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:56,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:56,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:56,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:56,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341816295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:56,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341816306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:56,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:56,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:56,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:56,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/1cae9f6d99984f2fb8cea0dfdcfdf631 2024-11-11T16:15:56,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b2dd3269dc454d3db58ad80d9e2f5103 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b2dd3269dc454d3db58ad80d9e2f5103 2024-11-11T16:15:56,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b2dd3269dc454d3db58ad80d9e2f5103, entries=200, sequenceid=343, filesize=39.0 K 2024-11-11T16:15:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/8b6827f814224299a29ef63f347ad22b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/8b6827f814224299a29ef63f347ad22b 2024-11-11T16:15:56,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/8b6827f814224299a29ef63f347ad22b, entries=150, sequenceid=343, filesize=12.0 K 2024-11-11T16:15:56,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/1cae9f6d99984f2fb8cea0dfdcfdf631 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1cae9f6d99984f2fb8cea0dfdcfdf631 2024-11-11T16:15:56,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1cae9f6d99984f2fb8cea0dfdcfdf631, entries=150, sequenceid=343, filesize=12.0 K 2024-11-11T16:15:56,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=114.05 KB/116790 for 79645af1f242c463cd05cc60d56c4bdc in 1488ms, sequenceid=343, compaction requested=false 2024-11-11T16:15:56,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:56,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T16:15:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:56,502 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:15:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:56,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111652bd80c633446e5b72c835ba57624e8_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341755106/Put/seqid=0 2024-11-11T16:15:56,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742228_1404 (size=12454) 2024-11-11T16:15:56,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:56,599 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111652bd80c633446e5b72c835ba57624e8_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111652bd80c633446e5b72c835ba57624e8_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/7d632f9169e043ab86341d423dfdbd50, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/7d632f9169e043ab86341d423dfdbd50 is 175, key is test_row_0/A:col10/1731341755106/Put/seqid=0 2024-11-11T16:15:56,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742229_1405 (size=31255) 2024-11-11T16:15:56,641 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/7d632f9169e043ab86341d423dfdbd50 2024-11-11T16:15:56,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/0969759805eb40e9966c669cc313eca6 is 50, key is test_row_0/B:col10/1731341755106/Put/seqid=0 2024-11-11T16:15:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742230_1406 (size=12301) 2024-11-11T16:15:57,122 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/0969759805eb40e9966c669cc313eca6 2024-11-11T16:15:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/90543d136b294c91a947bc9a977642df is 50, key is test_row_0/C:col10/1731341755106/Put/seqid=0 2024-11-11T16:15:57,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742231_1407 (size=12301) 2024-11-11T16:15:57,163 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/90543d136b294c91a947bc9a977642df 2024-11-11T16:15:57,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/7d632f9169e043ab86341d423dfdbd50 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/7d632f9169e043ab86341d423dfdbd50 2024-11-11T16:15:57,181 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/7d632f9169e043ab86341d423dfdbd50, entries=150, sequenceid=366, filesize=30.5 K 2024-11-11T16:15:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/0969759805eb40e9966c669cc313eca6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/0969759805eb40e9966c669cc313eca6 2024-11-11T16:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,186 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/0969759805eb40e9966c669cc313eca6, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T16:15:57,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/90543d136b294c91a947bc9a977642df as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/90543d136b294c91a947bc9a977642df 2024-11-11T16:15:57,191 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/90543d136b294c91a947bc9a977642df, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T16:15:57,192 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 79645af1f242c463cd05cc60d56c4bdc in 690ms, sequenceid=366, compaction requested=true 2024-11-11T16:15:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T16:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T16:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T16:15:57,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4830 sec 2024-11-11T16:15:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.4900 sec 2024-11-11T16:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:57,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:15:57,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:57,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:57,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:57,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:57,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:57,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:57,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118ade6da78db24a85a6282daec2ad96d5_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341757592/Put/seqid=0 2024-11-11T16:15:57,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742232_1408 (size=12454) 2024-11-11T16:15:57,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341757717, max timestamp=1731341757746Num uniques 859; 2024-11-11T16:15:57,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:57,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341757717, max timestamp=1731341757746Num uniques 859; 2024-11-11T16:15:57,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341757717, max timestamp=1731341757746Num uniques 859; 2024-11-11T16:15:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T16:15:57,876 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:15:57,877 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:15:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T16:15:57,879 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:15:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T16:15:57,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:15:57,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:15:57,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341817891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:57,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341817893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T16:15:58,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:58,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341818015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341818015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:58,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T16:15:58,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118ade6da78db24a85a6282daec2ad96d5_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118ade6da78db24a85a6282daec2ad96d5_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:58,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/79fca689e7ac4af19479c953fce2acd0, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:58,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/79fca689e7ac4af19479c953fce2acd0 is 175, key is test_row_0/A:col10/1731341757592/Put/seqid=0 2024-11-11T16:15:58,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742233_1409 (size=31255) 2024-11-11T16:15:58,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T16:15:58,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:58,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T16:15:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341818236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341818236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:58,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T16:15:58,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:58,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,443 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/79fca689e7ac4af19479c953fce2acd0 2024-11-11T16:15:58,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/135dc67f12a044d6a3b8161d1edf244f is 50, key is test_row_0/B:col10/1731341757592/Put/seqid=0 2024-11-11T16:15:58,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742234_1410 (size=12301) 2024-11-11T16:15:58,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/135dc67f12a044d6a3b8161d1edf244f 2024-11-11T16:15:58,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/6be071df0e834d3c957f365efa09cdf1 is 50, key is test_row_0/C:col10/1731341757592/Put/seqid=0 2024-11-11T16:15:58,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742235_1411 (size=12301) 2024-11-11T16:15:58,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T16:15:58,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:58,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T16:15:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341818556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341818556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:58,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T16:15:58,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T16:15:58,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:58,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:15:58,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/6be071df0e834d3c957f365efa09cdf1 2024-11-11T16:15:58,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/79fca689e7ac4af19479c953fce2acd0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/79fca689e7ac4af19479c953fce2acd0 2024-11-11T16:15:58,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/79fca689e7ac4af19479c953fce2acd0, entries=150, sequenceid=376, filesize=30.5 K 2024-11-11T16:15:58,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/135dc67f12a044d6a3b8161d1edf244f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/135dc67f12a044d6a3b8161d1edf244f 2024-11-11T16:15:58,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/135dc67f12a044d6a3b8161d1edf244f, entries=150, sequenceid=376, filesize=12.0 K 2024-11-11T16:15:58,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/6be071df0e834d3c957f365efa09cdf1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6be071df0e834d3c957f365efa09cdf1 2024-11-11T16:15:58,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6be071df0e834d3c957f365efa09cdf1, entries=150, sequenceid=376, filesize=12.0 K 2024-11-11T16:15:58,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 79645af1f242c463cd05cc60d56c4bdc in 1281ms, sequenceid=376, compaction requested=true 2024-11-11T16:15:58,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:15:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:58,897 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:58,897 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:15:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:15:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:58,898 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:58,898 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:58,898 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:15:58,898 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:15:58,898 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,898 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,898 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/c1d074b164124e97be51b5dd3fd9e2eb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b2dd3269dc454d3db58ad80d9e2f5103, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/7d632f9169e043ab86341d423dfdbd50, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/79fca689e7ac4af19479c953fce2acd0] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=131.0 K 2024-11-11T16:15:58,898 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/2ba0229ae1ca44aeb54037c0ffef15eb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/8b6827f814224299a29ef63f347ad22b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/0969759805eb40e9966c669cc313eca6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/135dc67f12a044d6a3b8161d1edf244f] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=48.5 K 2024-11-11T16:15:58,899 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,899 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/c1d074b164124e97be51b5dd3fd9e2eb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b2dd3269dc454d3db58ad80d9e2f5103, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/7d632f9169e043ab86341d423dfdbd50, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/79fca689e7ac4af19479c953fce2acd0] 2024-11-11T16:15:58,899 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba0229ae1ca44aeb54037c0ffef15eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731341752869 2024-11-11T16:15:58,899 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d074b164124e97be51b5dd3fd9e2eb, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731341752869 2024-11-11T16:15:58,899 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b6827f814224299a29ef63f347ad22b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731341753687 2024-11-11T16:15:58,900 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0969759805eb40e9966c669cc313eca6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341755076 2024-11-11T16:15:58,900 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2dd3269dc454d3db58ad80d9e2f5103, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731341753687 2024-11-11T16:15:58,900 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 135dc67f12a044d6a3b8161d1edf244f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731341757592 2024-11-11T16:15:58,900 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d632f9169e043ab86341d423dfdbd50, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341755076 2024-11-11T16:15:58,901 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79fca689e7ac4af19479c953fce2acd0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731341757592 2024-11-11T16:15:58,909 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:58,911 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/fc12b9bfd1384febb868a9b956bc4c77 is 50, key is test_row_0/B:col10/1731341757592/Put/seqid=0 2024-11-11T16:15:58,913 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:58,917 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411113cccf07ea0fb4c4491a40df6268ed45f_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:58,919 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411113cccf07ea0fb4c4491a40df6268ed45f_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:58,919 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113cccf07ea0fb4c4491a40df6268ed45f_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:58,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742236_1412 (size=12881) 2024-11-11T16:15:58,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742237_1413 (size=4469) 2024-11-11T16:15:58,933 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#349 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:58,934 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/9491c8824f7f495b903608b267e07dfa is 175, key is test_row_0/A:col10/1731341757592/Put/seqid=0 2024-11-11T16:15:58,936 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/fc12b9bfd1384febb868a9b956bc4c77 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/fc12b9bfd1384febb868a9b956bc4c77 2024-11-11T16:15:58,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742238_1414 (size=31835) 2024-11-11T16:15:58,941 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into fc12b9bfd1384febb868a9b956bc4c77(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:58,941 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:58,941 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=12, startTime=1731341758897; duration=0sec 2024-11-11T16:15:58,941 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:15:58,941 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:15:58,941 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:15:58,942 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:15:58,942 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:15:58,942 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,942 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/d11cd12fff66432aae13d4ed32fd9d9e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1cae9f6d99984f2fb8cea0dfdcfdf631, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/90543d136b294c91a947bc9a977642df, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6be071df0e834d3c957f365efa09cdf1] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=48.5 K 2024-11-11T16:15:58,943 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d11cd12fff66432aae13d4ed32fd9d9e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731341752869 2024-11-11T16:15:58,943 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cae9f6d99984f2fb8cea0dfdcfdf631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731341753687 2024-11-11T16:15:58,943 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 90543d136b294c91a947bc9a977642df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341755076 2024-11-11T16:15:58,944 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be071df0e834d3c957f365efa09cdf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731341757592 2024-11-11T16:15:58,954 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:15:58,954 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/defac8e66c734816be979d913e8f0be3 is 50, key is test_row_0/C:col10/1731341757592/Put/seqid=0 2024-11-11T16:15:58,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T16:15:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:58,959 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T16:15:58,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:58,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:58,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:58,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:58,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:58,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:15:58,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742239_1415 (size=12881) 2024-11-11T16:15:58,970 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/defac8e66c734816be979d913e8f0be3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/defac8e66c734816be979d913e8f0be3 2024-11-11T16:15:58,974 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into defac8e66c734816be979d913e8f0be3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:58,974 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:58,975 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=12, startTime=1731341758897; duration=0sec 2024-11-11T16:15:58,975 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:58,975 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:15:58,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ffb7bc4fb9964fbb80c1265743f1e0f7_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341757820/Put/seqid=0 2024-11-11T16:15:58,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742240_1416 (size=12454) 2024-11-11T16:15:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T16:15:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:59,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:15:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341819118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341819127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341819235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341819235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,344 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/9491c8824f7f495b903608b267e07dfa as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/9491c8824f7f495b903608b267e07dfa 2024-11-11T16:15:59,350 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into 9491c8824f7f495b903608b267e07dfa(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:15:59,350 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:59,350 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=12, startTime=1731341758897; duration=0sec 2024-11-11T16:15:59,350 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:15:59,350 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:15:59,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:15:59,397 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ffb7bc4fb9964fbb80c1265743f1e0f7_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ffb7bc4fb9964fbb80c1265743f1e0f7_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2b1e60d8e0c84411bf4f7f59ab475f9f, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:15:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2b1e60d8e0c84411bf4f7f59ab475f9f is 175, key is test_row_0/A:col10/1731341757820/Put/seqid=0 2024-11-11T16:15:59,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742241_1417 (size=31255) 2024-11-11T16:15:59,412 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2b1e60d8e0c84411bf4f7f59ab475f9f 2024-11-11T16:15:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/808b04b83ae74b71838ccf56f3d6f8e7 is 50, key is test_row_0/B:col10/1731341757820/Put/seqid=0 2024-11-11T16:15:59,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742242_1418 (size=12301) 2024-11-11T16:15:59,432 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/808b04b83ae74b71838ccf56f3d6f8e7 2024-11-11T16:15:59,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/e406ce37ed64425585e92a88c36a96e6 is 50, key is test_row_0/C:col10/1731341757820/Put/seqid=0 2024-11-11T16:15:59,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742243_1419 (size=12301) 2024-11-11T16:15:59,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341819456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341819456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341819776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:15:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341819776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:15:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:15:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:15:59,858 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/e406ce37ed64425585e92a88c36a96e6 2024-11-11T16:15:59,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2b1e60d8e0c84411bf4f7f59ab475f9f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2b1e60d8e0c84411bf4f7f59ab475f9f 2024-11-11T16:15:59,867 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2b1e60d8e0c84411bf4f7f59ab475f9f, entries=150, sequenceid=414, filesize=30.5 K 2024-11-11T16:15:59,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/808b04b83ae74b71838ccf56f3d6f8e7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/808b04b83ae74b71838ccf56f3d6f8e7 2024-11-11T16:15:59,871 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/808b04b83ae74b71838ccf56f3d6f8e7, entries=150, sequenceid=414, filesize=12.0 K 2024-11-11T16:15:59,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/e406ce37ed64425585e92a88c36a96e6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e406ce37ed64425585e92a88c36a96e6 2024-11-11T16:15:59,875 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e406ce37ed64425585e92a88c36a96e6, entries=150, sequenceid=414, filesize=12.0 K 2024-11-11T16:15:59,876 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 79645af1f242c463cd05cc60d56c4bdc in 917ms, sequenceid=414, compaction requested=false 2024-11-11T16:15:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:15:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:15:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T16:15:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T16:15:59,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T16:15:59,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9970 sec 2024-11-11T16:15:59,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.0030 sec 2024-11-11T16:15:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:15:59,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T16:15:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:15:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:15:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:15:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:15:59,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111153eab99d45db40a68d03931be55397b0_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341759125/Put/seqid=0 2024-11-11T16:15:59,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742244_1420 (size=14994) 2024-11-11T16:16:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T16:16:00,026 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:00,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T16:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T16:16:00,028 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:00,028 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:00,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:00,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341760040, max timestamp=1731341760059Num uniques 859; 2024-11-11T16:16:00,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341760040, max timestamp=1731341760059Num uniques 859; 2024-11-11T16:16:00,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:16:00,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:16:00,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731341760040, max timestamp=1731341760059Num uniques 859; 2024-11-11T16:16:00,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T16:16:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341820116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341820131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341820131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T16:16:00,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341820236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341820236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341820236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341820296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341820296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:00,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:00,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T16:16:00,392 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:00,396 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111153eab99d45db40a68d03931be55397b0_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111153eab99d45db40a68d03931be55397b0_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:00,397 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/0c7c849736b44b8f8e0df6e395c89565, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:00,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/0c7c849736b44b8f8e0df6e395c89565 is 175, key is test_row_0/A:col10/1731341759125/Put/seqid=0 2024-11-11T16:16:00,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742245_1421 (size=39949) 2024-11-11T16:16:00,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341820456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341820456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341820456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:00,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:00,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:00,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:00,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T16:16:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341820776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341820776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341820776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:00,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=427, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/0c7c849736b44b8f8e0df6e395c89565 2024-11-11T16:16:00,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/06cf190893464366ac21ba479033ccaa is 50, key is test_row_0/B:col10/1731341759125/Put/seqid=0 2024-11-11T16:16:00,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742246_1422 (size=12301) 2024-11-11T16:16:00,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:00,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T16:16:01,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/06cf190893464366ac21ba479033ccaa 2024-11-11T16:16:01,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/9412e82ac60d443b857ffee927e109c6 is 50, key is test_row_0/C:col10/1731341759125/Put/seqid=0 2024-11-11T16:16:01,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742247_1423 (size=12301) 2024-11-11T16:16:01,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:01,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:01,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341821296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341821296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341821296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:01,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341821316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:01,329 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:01,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341821326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:01,344 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_0' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:01,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:01,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:01,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:01,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:01,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:01,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/9412e82ac60d443b857ffee927e109c6 2024-11-11T16:16:01,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/0c7c849736b44b8f8e0df6e395c89565 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/0c7c849736b44b8f8e0df6e395c89565 2024-11-11T16:16:01,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/0c7c849736b44b8f8e0df6e395c89565, entries=200, sequenceid=427, filesize=39.0 K 2024-11-11T16:16:01,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/06cf190893464366ac21ba479033ccaa as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/06cf190893464366ac21ba479033ccaa 2024-11-11T16:16:01,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/06cf190893464366ac21ba479033ccaa, entries=150, sequenceid=427, filesize=12.0 K 2024-11-11T16:16:01,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/9412e82ac60d443b857ffee927e109c6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/9412e82ac60d443b857ffee927e109c6 2024-11-11T16:16:01,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/9412e82ac60d443b857ffee927e109c6, entries=150, sequenceid=427, filesize=12.0 K 2024-11-11T16:16:01,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 79645af1f242c463cd05cc60d56c4bdc in 1667ms, sequenceid=427, compaction requested=true 2024-11-11T16:16:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:01,649 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:01,649 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:01,650 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:01,650 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:16:01,650 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,650 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:01,650 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/9491c8824f7f495b903608b267e07dfa, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2b1e60d8e0c84411bf4f7f59ab475f9f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/0c7c849736b44b8f8e0df6e395c89565] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=100.6 K 2024-11-11T16:16:01,650 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:16:01,650 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,650 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/9491c8824f7f495b903608b267e07dfa, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2b1e60d8e0c84411bf4f7f59ab475f9f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/0c7c849736b44b8f8e0df6e395c89565] 2024-11-11T16:16:01,650 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,651 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/fc12b9bfd1384febb868a9b956bc4c77, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/808b04b83ae74b71838ccf56f3d6f8e7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/06cf190893464366ac21ba479033ccaa] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=36.6 K 2024-11-11T16:16:01,651 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9491c8824f7f495b903608b267e07dfa, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731341757592 2024-11-11T16:16:01,651 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting fc12b9bfd1384febb868a9b956bc4c77, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731341757592 2024-11-11T16:16:01,651 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 808b04b83ae74b71838ccf56f3d6f8e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731341757820 2024-11-11T16:16:01,652 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b1e60d8e0c84411bf4f7f59ab475f9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731341757820 2024-11-11T16:16:01,652 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 06cf190893464366ac21ba479033ccaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731341759117 2024-11-11T16:16:01,652 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7c849736b44b8f8e0df6e395c89565, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731341759087 2024-11-11T16:16:01,659 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:01,661 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:01,661 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/f9bc175587da480fabda1bb959c9d26e is 50, key is test_row_0/B:col10/1731341759125/Put/seqid=0 2024-11-11T16:16:01,663 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b5541b7086d64254ab615b7a0e7bbc2f_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:01,665 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b5541b7086d64254ab615b7a0e7bbc2f_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:01,665 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b5541b7086d64254ab615b7a0e7bbc2f_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:01,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742248_1424 (size=12983) 2024-11-11T16:16:01,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742249_1425 (size=4469) 2024-11-11T16:16:01,691 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/f9bc175587da480fabda1bb959c9d26e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f9bc175587da480fabda1bb959c9d26e 2024-11-11T16:16:01,696 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into f9bc175587da480fabda1bb959c9d26e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:01,696 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:01,696 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=13, startTime=1731341761649; duration=0sec 2024-11-11T16:16:01,696 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:01,696 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:16:01,696 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:01,697 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:01,697 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:16:01,697 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,697 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/defac8e66c734816be979d913e8f0be3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e406ce37ed64425585e92a88c36a96e6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/9412e82ac60d443b857ffee927e109c6] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=36.6 K 2024-11-11T16:16:01,698 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting defac8e66c734816be979d913e8f0be3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731341757592 2024-11-11T16:16:01,698 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e406ce37ed64425585e92a88c36a96e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731341757820 2024-11-11T16:16:01,698 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9412e82ac60d443b857ffee927e109c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731341759117 2024-11-11T16:16:01,704 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:01,705 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/02725175963049fda7d254ef07658e03 is 50, key is test_row_0/C:col10/1731341759125/Put/seqid=0 2024-11-11T16:16:01,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742250_1426 (size=12983) 2024-11-11T16:16:01,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T16:16:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:01,714 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T16:16:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:16:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:16:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:16:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:01,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d3bcfd5363964bab85742f5a24e0c55f_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341760129/Put/seqid=0 2024-11-11T16:16:01,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742251_1427 (size=12454) 2024-11-11T16:16:02,088 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:02,089 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/8d2464348036474d9c9a99496fbfcc63 is 175, key is test_row_0/A:col10/1731341759125/Put/seqid=0 2024-11-11T16:16:02,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742252_1428 (size=31937) 2024-11-11T16:16:02,113 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/02725175963049fda7d254ef07658e03 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02725175963049fda7d254ef07658e03 2024-11-11T16:16:02,117 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into 02725175963049fda7d254ef07658e03(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:02,117 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:02,117 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=13, startTime=1731341761649; duration=0sec 2024-11-11T16:16:02,117 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:02,117 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:16:02,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:02,129 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d3bcfd5363964bab85742f5a24e0c55f_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3bcfd5363964bab85742f5a24e0c55f_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/fe6a820280574032b190673bdcfb93c5, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/fe6a820280574032b190673bdcfb93c5 is 175, key is test_row_0/A:col10/1731341760129/Put/seqid=0 2024-11-11T16:16:02,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742253_1429 (size=31255) 2024-11-11T16:16:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T16:16:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:02,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341822381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341822381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341822382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341822486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341822486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,499 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/8d2464348036474d9c9a99496fbfcc63 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8d2464348036474d9c9a99496fbfcc63 2024-11-11T16:16:02,503 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into 8d2464348036474d9c9a99496fbfcc63(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:02,503 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:02,503 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=13, startTime=1731341761649; duration=0sec 2024-11-11T16:16:02,503 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:02,504 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:16:02,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341822506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,534 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/fe6a820280574032b190673bdcfb93c5 2024-11-11T16:16:02,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/9c4ba0c9e803455ca3914cf5d8251a8a is 50, key is test_row_0/B:col10/1731341760129/Put/seqid=0 2024-11-11T16:16:02,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742254_1430 (size=12301) 2024-11-11T16:16:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341822706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341822706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341822726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:02,946 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/9c4ba0c9e803455ca3914cf5d8251a8a 2024-11-11T16:16:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/0fc3d67683434664a9fe7624c6753fa1 is 50, key is test_row_0/C:col10/1731341760129/Put/seqid=0 2024-11-11T16:16:02,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742255_1431 (size=12301) 2024-11-11T16:16:03,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341823015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341823015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341823036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731341823336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,350 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,360 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/0fc3d67683434664a9fe7624c6753fa1 2024-11-11T16:16:03,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/fe6a820280574032b190673bdcfb93c5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fe6a820280574032b190673bdcfb93c5 2024-11-11T16:16:03,367 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fe6a820280574032b190673bdcfb93c5, entries=150, sequenceid=463, filesize=30.5 K 2024-11-11T16:16:03,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/9c4ba0c9e803455ca3914cf5d8251a8a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9c4ba0c9e803455ca3914cf5d8251a8a 2024-11-11T16:16:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,372 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9c4ba0c9e803455ca3914cf5d8251a8a, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T16:16:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/0fc3d67683434664a9fe7624c6753fa1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0fc3d67683434664a9fe7624c6753fa1 2024-11-11T16:16:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1731341823366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,375 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_0' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,378 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0fc3d67683434664a9fe7624c6753fa1, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T16:16:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,379 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=60.38 KB/61830 for 79645af1f242c463cd05cc60d56c4bdc in 1665ms, sequenceid=463, compaction requested=false 2024-11-11T16:16:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T16:16:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T16:16:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T16:16:03,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3530 sec 2024-11-11T16:16:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.3580 sec 2024-11-11T16:16:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:16:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:16:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d7dc4ea08494178b9eaea9d352313dc_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_1/A:col10/1731341763547/Put/seqid=0 2024-11-11T16:16:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742256_1432 (size=12454) 2024-11-11T16:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,595 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,599 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d7dc4ea08494178b9eaea9d352313dc_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d7dc4ea08494178b9eaea9d352313dc_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2c64a5e5992f42a18195d3582d109b28, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2c64a5e5992f42a18195d3582d109b28 is 175, key is test_row_1/A:col10/1731341763547/Put/seqid=0 2024-11-11T16:16:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742257_1433 (size=31251) 2024-11-11T16:16:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,608 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=478, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2c64a5e5992f42a18195d3582d109b28 2024-11-11T16:16:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/76996f2089f246a2ab9220faae808b79 is 50, key is test_row_1/B:col10/1731341763547/Put/seqid=0 2024-11-11T16:16:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742258_1434 (size=9857) 2024-11-11T16:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/76996f2089f246a2ab9220faae808b79 2024-11-11T16:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/006f3bfefd80431db940dae4156eee01 is 50, key is test_row_1/C:col10/1731341763547/Put/seqid=0 2024-11-11T16:16:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742259_1435 (size=9857) 2024-11-11T16:16:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:03,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341823738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341823740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341823745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341823855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341823856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:03,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341823866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/006f3bfefd80431db940dae4156eee01 2024-11-11T16:16:04,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/2c64a5e5992f42a18195d3582d109b28 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2c64a5e5992f42a18195d3582d109b28 2024-11-11T16:16:04,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2c64a5e5992f42a18195d3582d109b28, entries=150, sequenceid=478, filesize=30.5 K 2024-11-11T16:16:04,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/76996f2089f246a2ab9220faae808b79 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/76996f2089f246a2ab9220faae808b79 2024-11-11T16:16:04,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341824067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341824067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/76996f2089f246a2ab9220faae808b79, entries=100, sequenceid=478, filesize=9.6 K 2024-11-11T16:16:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/006f3bfefd80431db940dae4156eee01 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/006f3bfefd80431db940dae4156eee01 2024-11-11T16:16:04,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/006f3bfefd80431db940dae4156eee01, entries=100, sequenceid=478, filesize=9.6 K 2024-11-11T16:16:04,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 79645af1f242c463cd05cc60d56c4bdc in 527ms, sequenceid=478, compaction requested=true 2024-11-11T16:16:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:04,078 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:04,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:04,078 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:04,079 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:04,079 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:04,079 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:16:04,079 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:16:04,079 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,079 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,079 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8d2464348036474d9c9a99496fbfcc63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fe6a820280574032b190673bdcfb93c5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2c64a5e5992f42a18195d3582d109b28] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=92.2 K 2024-11-11T16:16:04,079 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f9bc175587da480fabda1bb959c9d26e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9c4ba0c9e803455ca3914cf5d8251a8a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/76996f2089f246a2ab9220faae808b79] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=34.3 K 2024-11-11T16:16:04,079 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,079 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8d2464348036474d9c9a99496fbfcc63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fe6a820280574032b190673bdcfb93c5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2c64a5e5992f42a18195d3582d109b28] 2024-11-11T16:16:04,080 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f9bc175587da480fabda1bb959c9d26e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731341759117 2024-11-11T16:16:04,080 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d2464348036474d9c9a99496fbfcc63, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731341759117 2024-11-11T16:16:04,080 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4ba0c9e803455ca3914cf5d8251a8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731341760113 2024-11-11T16:16:04,080 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe6a820280574032b190673bdcfb93c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731341760113 2024-11-11T16:16:04,081 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 76996f2089f246a2ab9220faae808b79, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731341762318 2024-11-11T16:16:04,081 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c64a5e5992f42a18195d3582d109b28, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731341762318 2024-11-11T16:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:04,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:16:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:16:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:16:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:16:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:04,089 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:04,090 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/18ecb6ab7ed04360ae3fc2abfef508c4 is 50, key is test_row_0/B:col10/1731341760129/Put/seqid=0 2024-11-11T16:16:04,091 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:04,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b77ef60a529645059abdc6fa05551fe8_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341764083/Put/seqid=0 2024-11-11T16:16:04,094 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b6521abaa3454163955bbfceaf26b161_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:04,096 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b6521abaa3454163955bbfceaf26b161_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:04,096 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b6521abaa3454163955bbfceaf26b161_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:04,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742261_1437 (size=14994) 2024-11-11T16:16:04,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742260_1436 (size=13085) 2024-11-11T16:16:04,098 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:04,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b77ef60a529645059abdc6fa05551fe8_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b77ef60a529645059abdc6fa05551fe8_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:04,103 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/58edd41050f24895aa35510bef4ee543, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:04,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/58edd41050f24895aa35510bef4ee543 is 175, key is test_row_0/A:col10/1731341764083/Put/seqid=0 2024-11-11T16:16:04,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742262_1438 (size=4469) 2024-11-11T16:16:04,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742263_1439 (size=39949) 2024-11-11T16:16:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341824164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T16:16:04,196 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:04,196 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-11T16:16:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T16:16:04,198 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:04,198 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:04,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341824286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T16:16:04,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T16:16:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341824376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341824386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,504 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/18ecb6ab7ed04360ae3fc2abfef508c4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/18ecb6ab7ed04360ae3fc2abfef508c4 2024-11-11T16:16:04,508 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into 18ecb6ab7ed04360ae3fc2abfef508c4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:04,508 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:04,509 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=13, startTime=1731341764078; duration=0sec 2024-11-11T16:16:04,509 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:04,509 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:16:04,509 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:04,510 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:04,510 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#367 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:04,510 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:16:04,510 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,511 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02725175963049fda7d254ef07658e03, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0fc3d67683434664a9fe7624c6753fa1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/006f3bfefd80431db940dae4156eee01] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=34.3 K 2024-11-11T16:16:04,511 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/3ae29482fb3b4dc1a4bd46887eaef77e is 175, key is test_row_0/A:col10/1731341760129/Put/seqid=0 2024-11-11T16:16:04,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T16:16:04,511 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 02725175963049fda7d254ef07658e03, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731341759117 2024-11-11T16:16:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,512 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc3d67683434664a9fe7624c6753fa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731341760113 2024-11-11T16:16:04,513 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 006f3bfefd80431db940dae4156eee01, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731341762318 2024-11-11T16:16:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,515 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=498, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/58edd41050f24895aa35510bef4ee543 2024-11-11T16:16:04,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742264_1440 (size=32146) 2024-11-11T16:16:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T16:16:04,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341824506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,522 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:04,522 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/21b5cb99432d4154b7f55d0a17bff719 is 50, key is test_row_0/C:col10/1731341760129/Put/seqid=0 2024-11-11T16:16:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/6bfe6cacf4fc4f819412643658efc4b2 is 50, key is test_row_0/B:col10/1731341764083/Put/seqid=0 2024-11-11T16:16:04,526 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/3ae29482fb3b4dc1a4bd46887eaef77e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3ae29482fb3b4dc1a4bd46887eaef77e 2024-11-11T16:16:04,533 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into 3ae29482fb3b4dc1a4bd46887eaef77e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:04,533 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:04,533 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=13, startTime=1731341764078; duration=0sec 2024-11-11T16:16:04,533 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:04,533 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:16:04,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742266_1442 (size=12301) 2024-11-11T16:16:04,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742265_1441 (size=13085) 2024-11-11T16:16:04,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T16:16:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T16:16:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T16:16:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341824825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341824896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341824905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:04,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/6bfe6cacf4fc4f819412643658efc4b2 2024-11-11T16:16:04,949 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/21b5cb99432d4154b7f55d0a17bff719 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/21b5cb99432d4154b7f55d0a17bff719 2024-11-11T16:16:04,954 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into 21b5cb99432d4154b7f55d0a17bff719(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:04,954 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:04,954 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=13, startTime=1731341764078; duration=0sec 2024-11-11T16:16:04,954 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:04,954 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:16:04,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/34eef011b64c4feb80ccc3d1d68b1a43 is 50, key is test_row_0/C:col10/1731341764083/Put/seqid=0 2024-11-11T16:16:04,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T16:16:04,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:04,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:04,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742267_1443 (size=12301) 2024-11-11T16:16:04,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/34eef011b64c4feb80ccc3d1d68b1a43 2024-11-11T16:16:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/58edd41050f24895aa35510bef4ee543 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/58edd41050f24895aa35510bef4ee543 2024-11-11T16:16:04,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/58edd41050f24895aa35510bef4ee543, entries=200, sequenceid=498, filesize=39.0 K 2024-11-11T16:16:04,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/6bfe6cacf4fc4f819412643658efc4b2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6bfe6cacf4fc4f819412643658efc4b2 2024-11-11T16:16:04,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6bfe6cacf4fc4f819412643658efc4b2, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T16:16:04,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/34eef011b64c4feb80ccc3d1d68b1a43 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/34eef011b64c4feb80ccc3d1d68b1a43 2024-11-11T16:16:04,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/34eef011b64c4feb80ccc3d1d68b1a43, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T16:16:04,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 79645af1f242c463cd05cc60d56c4bdc in 910ms, sequenceid=498, compaction requested=false 2024-11-11T16:16:04,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:05,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T16:16:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:05,126 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T16:16:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:16:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:16:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:05,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:16:05,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118f6222d17eea42daac0a75fff4fb0b6d_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341764154/Put/seqid=0 2024-11-11T16:16:05,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742268_1444 (size=12454) 2024-11-11T16:16:05,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:05,143 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118f6222d17eea42daac0a75fff4fb0b6d_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118f6222d17eea42daac0a75fff4fb0b6d_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:05,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/3e81797f2f5e4ef88e2753dc867e0121, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:05,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/3e81797f2f5e4ef88e2753dc867e0121 is 175, key is test_row_0/A:col10/1731341764154/Put/seqid=0 2024-11-11T16:16:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742269_1445 (size=31255) 2024-11-11T16:16:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T16:16:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:05,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341825495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:05,551 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=513, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/3e81797f2f5e4ef88e2753dc867e0121 2024-11-11T16:16:05,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/02c87b175d36471abe4b10f032ced24d is 50, key is test_row_0/B:col10/1731341764154/Put/seqid=0 2024-11-11T16:16:05,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742270_1446 (size=12301) 2024-11-11T16:16:05,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341825606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:05,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54920 deadline: 1731341825815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:05,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54966 deadline: 1731341825925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:05,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54978 deadline: 1731341825926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:05,936 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_0' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,936 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 79645af1f242c463cd05cc60d56c4bdc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:05,962 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/02c87b175d36471abe4b10f032ced24d 2024-11-11T16:16:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/be34c96e5cb04585a86d0da6860f3e3b is 50, key is test_row_0/C:col10/1731341764154/Put/seqid=0 2024-11-11T16:16:06,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742271_1447 (size=12301) 2024-11-11T16:16:06,020 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/be34c96e5cb04585a86d0da6860f3e3b 2024-11-11T16:16:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/3e81797f2f5e4ef88e2753dc867e0121 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3e81797f2f5e4ef88e2753dc867e0121 2024-11-11T16:16:06,051 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3e81797f2f5e4ef88e2753dc867e0121, entries=150, sequenceid=513, filesize=30.5 K 2024-11-11T16:16:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/02c87b175d36471abe4b10f032ced24d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/02c87b175d36471abe4b10f032ced24d 2024-11-11T16:16:06,064 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/02c87b175d36471abe4b10f032ced24d, entries=150, sequenceid=513, filesize=12.0 K 2024-11-11T16:16:06,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/be34c96e5cb04585a86d0da6860f3e3b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/be34c96e5cb04585a86d0da6860f3e3b 2024-11-11T16:16:06,095 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/be34c96e5cb04585a86d0da6860f3e3b, entries=150, sequenceid=513, filesize=12.0 K 2024-11-11T16:16:06,096 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 79645af1f242c463cd05cc60d56c4bdc in 970ms, sequenceid=513, compaction requested=true 2024-11-11T16:16:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-11T16:16:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-11T16:16:06,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T16:16:06,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-11-11T16:16:06,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.9040 sec 2024-11-11T16:16:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:06,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341765465, max timestamp=1731341766140Num uniques 859; 2024-11-11T16:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341765465, max timestamp=1731341766140Num uniques 859; 2024-11-11T16:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341765465, max timestamp=1731341766140Num uniques 859; 2024-11-11T16:16:06,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:06,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-11T16:16:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:16:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:16:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:16:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:06,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111bfee089941e41438aa3e0d7cec75207_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341765481/Put/seqid=0 2024-11-11T16:16:06,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742272_1448 (size=14994) 2024-11-11T16:16:06,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:06,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111bfee089941e41438aa3e0d7cec75207_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111bfee089941e41438aa3e0d7cec75207_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:06,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/1a9248b582bc4ead844216211deb8fc0, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:06,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/1a9248b582bc4ead844216211deb8fc0 is 175, key is test_row_0/A:col10/1731341765481/Put/seqid=0 2024-11-11T16:16:06,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742273_1449 (size=39949) 2024-11-11T16:16:06,285 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=534, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/1a9248b582bc4ead844216211deb8fc0 2024-11-11T16:16:06,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/094a5bbb438649eb97fc35e3f2f9855c is 50, key is test_row_0/B:col10/1731341765481/Put/seqid=0 2024-11-11T16:16:06,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742274_1450 (size=12301) 2024-11-11T16:16:06,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/094a5bbb438649eb97fc35e3f2f9855c 2024-11-11T16:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T16:16:06,357 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:06,359 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T16:16:06,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c745a88716e644f18e64295ce5105dd7 is 50, key is test_row_0/C:col10/1731341765481/Put/seqid=0 2024-11-11T16:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T16:16:06,361 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:06,362 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:06,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:06,365 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T16:16:06,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742275_1451 (size=12301) 2024-11-11T16:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T16:16:06,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T16:16:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:06,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341766448, max timestamp=1731341766523Num uniques 859; 2024-11-11T16:16:06,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:06,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341766448, max timestamp=1731341766523Num uniques 859; 2024-11-11T16:16:06,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:06,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341766448, max timestamp=1731341766523Num uniques 859; 2024-11-11T16:16:06,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:06,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T16:16:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:06,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:06,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T16:16:06,763 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-11T16:16:06,763 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:06,763 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,763 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-11T16:16:06,763 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:06,763 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:06,763 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,763 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:06,768 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-11-11T16:16:06,768 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:06,768 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,768 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:06,773 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-11T16:16:06,773 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:06,773 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,773 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:06,777 INFO [Thread-1575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1575. 2024-11-11T16:16:06,777 DEBUG [Thread-1575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:06,777 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,777 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:06,780 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-11-11T16:16:06,780 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:06,780 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,780 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:06,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:06,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c745a88716e644f18e64295ce5105dd7 2024-11-11T16:16:06,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/1a9248b582bc4ead844216211deb8fc0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/1a9248b582bc4ead844216211deb8fc0 2024-11-11T16:16:06,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/1a9248b582bc4ead844216211deb8fc0, entries=200, sequenceid=534, filesize=39.0 K 2024-11-11T16:16:06,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/094a5bbb438649eb97fc35e3f2f9855c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/094a5bbb438649eb97fc35e3f2f9855c 2024-11-11T16:16:06,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/094a5bbb438649eb97fc35e3f2f9855c, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T16:16:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c745a88716e644f18e64295ce5105dd7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c745a88716e644f18e64295ce5105dd7 2024-11-11T16:16:06,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c745a88716e644f18e64295ce5105dd7, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T16:16:06,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=100.63 KB/103050 for 79645af1f242c463cd05cc60d56c4bdc in 651ms, sequenceid=534, compaction requested=true 2024-11-11T16:16:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:06,799 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 79645af1f242c463cd05cc60d56c4bdc:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:06,799 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:06,800 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143299 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:06,800 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/A is initiating minor compaction (all files) 2024-11-11T16:16:06,800 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/A in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,800 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3ae29482fb3b4dc1a4bd46887eaef77e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/58edd41050f24895aa35510bef4ee543, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3e81797f2f5e4ef88e2753dc867e0121, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/1a9248b582bc4ead844216211deb8fc0] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=139.9 K 2024-11-11T16:16:06,800 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,800 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3ae29482fb3b4dc1a4bd46887eaef77e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/58edd41050f24895aa35510bef4ee543, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3e81797f2f5e4ef88e2753dc867e0121, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/1a9248b582bc4ead844216211deb8fc0] 2024-11-11T16:16:06,801 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:06,801 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/B is initiating minor compaction (all files) 2024-11-11T16:16:06,801 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/B in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,801 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae29482fb3b4dc1a4bd46887eaef77e, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731341760129 2024-11-11T16:16:06,801 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/18ecb6ab7ed04360ae3fc2abfef508c4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6bfe6cacf4fc4f819412643658efc4b2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/02c87b175d36471abe4b10f032ced24d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/094a5bbb438649eb97fc35e3f2f9855c] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=48.8 K 2024-11-11T16:16:06,816 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ecb6ab7ed04360ae3fc2abfef508c4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731341760129 2024-11-11T16:16:06,816 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58edd41050f24895aa35510bef4ee543, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731341763703 2024-11-11T16:16:06,820 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bfe6cacf4fc4f819412643658efc4b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731341763703 2024-11-11T16:16:06,821 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e81797f2f5e4ef88e2753dc867e0121, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731341764141 2024-11-11T16:16:06,831 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c87b175d36471abe4b10f032ced24d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731341764141 2024-11-11T16:16:06,834 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a9248b582bc4ead844216211deb8fc0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731341765465 2024-11-11T16:16:06,834 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 094a5bbb438649eb97fc35e3f2f9855c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731341765465 2024-11-11T16:16:06,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T16:16:06,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,842 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=100.63 KB heapSize=285.38 KB 2024-11-11T16:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f1bb367e8b4044c7aa7a70d5a05c3e74_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341766736/Put/seqid=0 2024-11-11T16:16:06,890 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:06,890 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/6411ab3d58744004b42a7de8b11d8c8a is 50, key is test_row_0/B:col10/1731341765481/Put/seqid=0 2024-11-11T16:16:06,896 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742276_1452 (size=12454) 2024-11-11T16:16:06,899 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111493ec6ead40b40ecab1f48e32acb4011_79645af1f242c463cd05cc60d56c4bdc store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:06,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742277_1453 (size=13221) 2024-11-11T16:16:06,902 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111493ec6ead40b40ecab1f48e32acb4011_79645af1f242c463cd05cc60d56c4bdc, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:06,902 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111493ec6ead40b40ecab1f48e32acb4011_79645af1f242c463cd05cc60d56c4bdc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:06,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742278_1454 (size=4469) 2024-11-11T16:16:06,907 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#A#compaction#380 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:06,908 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/30150e4834b34a20ad4e1f693732cc14 is 175, key is test_row_0/A:col10/1731341765481/Put/seqid=0 2024-11-11T16:16:06,908 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/6411ab3d58744004b42a7de8b11d8c8a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6411ab3d58744004b42a7de8b11d8c8a 2024-11-11T16:16:06,913 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/B of 79645af1f242c463cd05cc60d56c4bdc into 6411ab3d58744004b42a7de8b11d8c8a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:06,913 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:06,913 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/B, priority=12, startTime=1731341766799; duration=0sec 2024-11-11T16:16:06,913 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:06,913 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:B 2024-11-11T16:16:06,913 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:06,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742279_1455 (size=32175) 2024-11-11T16:16:06,914 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:06,914 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 79645af1f242c463cd05cc60d56c4bdc/C is initiating minor compaction (all files) 2024-11-11T16:16:06,914 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 79645af1f242c463cd05cc60d56c4bdc/C in TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:06,915 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/21b5cb99432d4154b7f55d0a17bff719, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/34eef011b64c4feb80ccc3d1d68b1a43, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/be34c96e5cb04585a86d0da6860f3e3b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c745a88716e644f18e64295ce5105dd7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp, totalSize=48.8 K 2024-11-11T16:16:06,915 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b5cb99432d4154b7f55d0a17bff719, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731341760129 2024-11-11T16:16:06,915 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 34eef011b64c4feb80ccc3d1d68b1a43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731341763703 2024-11-11T16:16:06,915 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting be34c96e5cb04585a86d0da6860f3e3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731341764141 2024-11-11T16:16:06,916 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c745a88716e644f18e64295ce5105dd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731341765465 2024-11-11T16:16:06,924 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 79645af1f242c463cd05cc60d56c4bdc#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:06,925 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/610c85266beb428c9231e884428aa9f0 is 50, key is test_row_0/C:col10/1731341765481/Put/seqid=0 2024-11-11T16:16:06,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742280_1456 (size=13221) 2024-11-11T16:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T16:16:07,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:07,302 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f1bb367e8b4044c7aa7a70d5a05c3e74_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f1bb367e8b4044c7aa7a70d5a05c3e74_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b5e849b547cf44048e8bc5f1e22cb1dc, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:07,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b5e849b547cf44048e8bc5f1e22cb1dc is 175, key is test_row_0/A:col10/1731341766736/Put/seqid=0 2024-11-11T16:16:07,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742281_1457 (size=31255) 2024-11-11T16:16:07,319 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/30150e4834b34a20ad4e1f693732cc14 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/30150e4834b34a20ad4e1f693732cc14 2024-11-11T16:16:07,323 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/A of 79645af1f242c463cd05cc60d56c4bdc into 30150e4834b34a20ad4e1f693732cc14(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:07,323 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:07,323 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/A, priority=12, startTime=1731341766799; duration=0sec 2024-11-11T16:16:07,323 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:07,323 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:A 2024-11-11T16:16:07,332 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/610c85266beb428c9231e884428aa9f0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/610c85266beb428c9231e884428aa9f0 2024-11-11T16:16:07,336 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 79645af1f242c463cd05cc60d56c4bdc/C of 79645af1f242c463cd05cc60d56c4bdc into 610c85266beb428c9231e884428aa9f0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:07,336 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:07,336 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., storeName=79645af1f242c463cd05cc60d56c4bdc/C, priority=12, startTime=1731341766799; duration=0sec 2024-11-11T16:16:07,336 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:07,336 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 79645af1f242c463cd05cc60d56c4bdc:C 2024-11-11T16:16:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:07,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. as already flushing 2024-11-11T16:16:07,388 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-11T16:16:07,388 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:07,389 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,389 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:07,398 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-11T16:16:07,398 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:07,398 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,398 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T16:16:07,708 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=566, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b5e849b547cf44048e8bc5f1e22cb1dc 2024-11-11T16:16:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/5aad01123a2b4e57a4cc9f37aee0675b is 50, key is test_row_0/B:col10/1731341766736/Put/seqid=0 2024-11-11T16:16:07,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742282_1458 (size=12301) 2024-11-11T16:16:07,947 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-11T16:16:07,948 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:07,948 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,948 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:07,968 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-11T16:16:07,968 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:07,968 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,969 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:07,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:08,117 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/5aad01123a2b4e57a4cc9f37aee0675b 2024-11-11T16:16:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c9ee4c39f0b74ac7a3cf63dd18a6abc2 is 50, key is test_row_0/C:col10/1731341766736/Put/seqid=0 2024-11-11T16:16:08,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742283_1459 (size=12301) 2024-11-11T16:16:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T16:16:08,527 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c9ee4c39f0b74ac7a3cf63dd18a6abc2 2024-11-11T16:16:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/b5e849b547cf44048e8bc5f1e22cb1dc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b5e849b547cf44048e8bc5f1e22cb1dc 2024-11-11T16:16:08,534 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b5e849b547cf44048e8bc5f1e22cb1dc, entries=150, sequenceid=566, filesize=30.5 K 2024-11-11T16:16:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/5aad01123a2b4e57a4cc9f37aee0675b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/5aad01123a2b4e57a4cc9f37aee0675b 2024-11-11T16:16:08,537 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/5aad01123a2b4e57a4cc9f37aee0675b, entries=150, sequenceid=566, filesize=12.0 K 2024-11-11T16:16:08,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/c9ee4c39f0b74ac7a3cf63dd18a6abc2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c9ee4c39f0b74ac7a3cf63dd18a6abc2 2024-11-11T16:16:08,540 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c9ee4c39f0b74ac7a3cf63dd18a6abc2, entries=150, sequenceid=566, filesize=12.0 K 2024-11-11T16:16:08,541 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~285.33 KB/292176, currentSize=26.84 KB/27480 for 79645af1f242c463cd05cc60d56c4bdc in 1699ms, sequenceid=566, compaction requested=false 2024-11-11T16:16:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 79645af1f242c463cd05cc60d56c4bdc: 2024-11-11T16:16:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T16:16:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T16:16:08,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T16:16:08,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1790 sec 2024-11-11T16:16:08,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1850 sec 2024-11-11T16:16:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T16:16:10,516 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 786 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2349 rows 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 790 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2360 rows 2024-11-11T16:16:10,516 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 781 2024-11-11T16:16:10,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2343 rows 2024-11-11T16:16:10,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 779 2024-11-11T16:16:10,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2337 rows 2024-11-11T16:16:10,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 791 2024-11-11T16:16:10,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2367 rows 2024-11-11T16:16:10,517 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:16:10,517 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:16:10,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:10,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:10,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:10,518 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T16:16:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T16:16:10,521 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341770521"}]},"ts":"1731341770521"} 2024-11-11T16:16:10,522 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T16:16:10,522 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T16:16:10,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:16:10,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, UNASSIGN}] 2024-11-11T16:16:10,525 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, UNASSIGN 2024-11-11T16:16:10,526 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:10,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, UNASSIGN because future has completed 2024-11-11T16:16:10,527 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:16:10,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:16:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T16:16:10,679 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:10,679 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:16:10,679 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 79645af1f242c463cd05cc60d56c4bdc, disabling compactions & flushes 2024-11-11T16:16:10,679 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:10,679 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:10,679 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. after waiting 0 ms 2024-11-11T16:16:10,679 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:10,680 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 79645af1f242c463cd05cc60d56c4bdc 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T16:16:10,680 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=A 2024-11-11T16:16:10,680 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:10,680 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=B 2024-11-11T16:16:10,680 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:10,680 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 79645af1f242c463cd05cc60d56c4bdc, store=C 2024-11-11T16:16:10,680 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:10,686 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112272e2d80cf346e7a7ec1a2dfb87f17d_79645af1f242c463cd05cc60d56c4bdc is 50, key is test_row_0/A:col10/1731341767967/Put/seqid=0 2024-11-11T16:16:10,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742284_1460 (size=9914) 2024-11-11T16:16:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T16:16:11,090 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:11,093 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112272e2d80cf346e7a7ec1a2dfb87f17d_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112272e2d80cf346e7a7ec1a2dfb87f17d_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:11,094 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/8920466f96c046949cd630cccb65fcb8, store: [table=TestAcidGuarantees family=A region=79645af1f242c463cd05cc60d56c4bdc] 2024-11-11T16:16:11,095 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/8920466f96c046949cd630cccb65fcb8 is 175, key is test_row_0/A:col10/1731341767967/Put/seqid=0 2024-11-11T16:16:11,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742285_1461 (size=22561) 2024-11-11T16:16:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T16:16:11,498 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=576, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/8920466f96c046949cd630cccb65fcb8 2024-11-11T16:16:11,505 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/3a094cc81aed482382ce15dea187cce0 is 50, key is test_row_0/B:col10/1731341767967/Put/seqid=0 2024-11-11T16:16:11,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742286_1462 (size=9857) 2024-11-11T16:16:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T16:16:11,909 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/3a094cc81aed482382ce15dea187cce0 2024-11-11T16:16:11,915 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/2d5b58d9507640bda7f44e4bf96158ef is 50, key is test_row_0/C:col10/1731341767967/Put/seqid=0 2024-11-11T16:16:11,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742287_1463 (size=9857) 2024-11-11T16:16:12,147 WARN [regionserver/16b413a53992:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-11T16:16:12,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/262054fb947946678a22bb6691ea7989, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/92376040c8384af3973937c102a55b63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b202beb8563545e9a86c4118b03451af, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/339101bb433949219ea4d92d0f68b4cc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/91996c89cfd44cddb4b23a0cc81d3cbd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a31ca8961a9540c3a83536a54dee9c2f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a2d62d345b4444188818de958b3af689, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fa5babdf25284d59b473499a75f849ee, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2ef2ac2cb02a4bccb62c5f5bcdefa101, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/163a1f486d7a4b178e7b8a995699447a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/d73843cee6a24917a4614a3ae9012ca1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/78aa56763bd34387bb242c96dff16e95, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/4979fa82b3a14b87b3e76645d9d609f4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/c1d074b164124e97be51b5dd3fd9e2eb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b2dd3269dc454d3db58ad80d9e2f5103, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/7d632f9169e043ab86341d423dfdbd50, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/9491c8824f7f495b903608b267e07dfa, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/79fca689e7ac4af19479c953fce2acd0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2b1e60d8e0c84411bf4f7f59ab475f9f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/0c7c849736b44b8f8e0df6e395c89565, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8d2464348036474d9c9a99496fbfcc63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fe6a820280574032b190673bdcfb93c5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3ae29482fb3b4dc1a4bd46887eaef77e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2c64a5e5992f42a18195d3582d109b28, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/58edd41050f24895aa35510bef4ee543, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3e81797f2f5e4ef88e2753dc867e0121, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/1a9248b582bc4ead844216211deb8fc0] to archive 2024-11-11T16:16:12,149 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:16:12,151 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/262054fb947946678a22bb6691ea7989 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/262054fb947946678a22bb6691ea7989 2024-11-11T16:16:12,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/92376040c8384af3973937c102a55b63 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/92376040c8384af3973937c102a55b63 2024-11-11T16:16:12,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b202beb8563545e9a86c4118b03451af to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b202beb8563545e9a86c4118b03451af 2024-11-11T16:16:12,154 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/339101bb433949219ea4d92d0f68b4cc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/339101bb433949219ea4d92d0f68b4cc 2024-11-11T16:16:12,154 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/91996c89cfd44cddb4b23a0cc81d3cbd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/91996c89cfd44cddb4b23a0cc81d3cbd 2024-11-11T16:16:12,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a31ca8961a9540c3a83536a54dee9c2f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a31ca8961a9540c3a83536a54dee9c2f 2024-11-11T16:16:12,156 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a2d62d345b4444188818de958b3af689 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/a2d62d345b4444188818de958b3af689 2024-11-11T16:16:12,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fa5babdf25284d59b473499a75f849ee to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fa5babdf25284d59b473499a75f849ee 2024-11-11T16:16:12,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2ef2ac2cb02a4bccb62c5f5bcdefa101 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2ef2ac2cb02a4bccb62c5f5bcdefa101 2024-11-11T16:16:12,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/163a1f486d7a4b178e7b8a995699447a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/163a1f486d7a4b178e7b8a995699447a 2024-11-11T16:16:12,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/d73843cee6a24917a4614a3ae9012ca1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/d73843cee6a24917a4614a3ae9012ca1 2024-11-11T16:16:12,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/78aa56763bd34387bb242c96dff16e95 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/78aa56763bd34387bb242c96dff16e95 2024-11-11T16:16:12,162 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/4979fa82b3a14b87b3e76645d9d609f4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/4979fa82b3a14b87b3e76645d9d609f4 2024-11-11T16:16:12,163 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/c1d074b164124e97be51b5dd3fd9e2eb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/c1d074b164124e97be51b5dd3fd9e2eb 2024-11-11T16:16:12,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b2dd3269dc454d3db58ad80d9e2f5103 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b2dd3269dc454d3db58ad80d9e2f5103 2024-11-11T16:16:12,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/7d632f9169e043ab86341d423dfdbd50 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/7d632f9169e043ab86341d423dfdbd50 2024-11-11T16:16:12,166 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/9491c8824f7f495b903608b267e07dfa to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/9491c8824f7f495b903608b267e07dfa 2024-11-11T16:16:12,167 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/79fca689e7ac4af19479c953fce2acd0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/79fca689e7ac4af19479c953fce2acd0 2024-11-11T16:16:12,168 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2b1e60d8e0c84411bf4f7f59ab475f9f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2b1e60d8e0c84411bf4f7f59ab475f9f 2024-11-11T16:16:12,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/0c7c849736b44b8f8e0df6e395c89565 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/0c7c849736b44b8f8e0df6e395c89565 2024-11-11T16:16:12,170 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8d2464348036474d9c9a99496fbfcc63 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8d2464348036474d9c9a99496fbfcc63 2024-11-11T16:16:12,171 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fe6a820280574032b190673bdcfb93c5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/fe6a820280574032b190673bdcfb93c5 2024-11-11T16:16:12,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3ae29482fb3b4dc1a4bd46887eaef77e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3ae29482fb3b4dc1a4bd46887eaef77e 2024-11-11T16:16:12,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2c64a5e5992f42a18195d3582d109b28 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/2c64a5e5992f42a18195d3582d109b28 2024-11-11T16:16:12,173 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/58edd41050f24895aa35510bef4ee543 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/58edd41050f24895aa35510bef4ee543 2024-11-11T16:16:12,174 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3e81797f2f5e4ef88e2753dc867e0121 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/3e81797f2f5e4ef88e2753dc867e0121 2024-11-11T16:16:12,175 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/1a9248b582bc4ead844216211deb8fc0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/1a9248b582bc4ead844216211deb8fc0 2024-11-11T16:16:12,178 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f680cef66d8d494d953f3130efada563, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/d5412f2bba1c4b51849aba55560eed1e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/33ffdbc6f1c142b98ab56fcf9dc1aad9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/c37ba6bd73544695bf63a80a9873b9e7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/43f0453b7c1f48f08135cba5426a7b39, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3201b186993c46d7bdf61e8c1506f212, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/4a10e0ceef2c4b468d2bb92ac7f2be63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9077c4c321e84046b236d208243bc56d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/29176ecba3414a04911b54d9b39e08b0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/e7346c0b53db4b3d9ce2c1d10acbc7ba, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/70ce831d62e4442e98bb315fedb954e7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/25dc7b494c5748069d3c2a3a9ae1bb74, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/2ba0229ae1ca44aeb54037c0ffef15eb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/82425fc9517747fe8a052dccdc48d133, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/8b6827f814224299a29ef63f347ad22b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/0969759805eb40e9966c669cc313eca6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/fc12b9bfd1384febb868a9b956bc4c77, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/135dc67f12a044d6a3b8161d1edf244f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/808b04b83ae74b71838ccf56f3d6f8e7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f9bc175587da480fabda1bb959c9d26e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/06cf190893464366ac21ba479033ccaa, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9c4ba0c9e803455ca3914cf5d8251a8a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/18ecb6ab7ed04360ae3fc2abfef508c4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/76996f2089f246a2ab9220faae808b79, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6bfe6cacf4fc4f819412643658efc4b2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/02c87b175d36471abe4b10f032ced24d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/094a5bbb438649eb97fc35e3f2f9855c] to archive 2024-11-11T16:16:12,179 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:16:12,180 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f680cef66d8d494d953f3130efada563 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f680cef66d8d494d953f3130efada563 2024-11-11T16:16:12,181 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/d5412f2bba1c4b51849aba55560eed1e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/d5412f2bba1c4b51849aba55560eed1e 2024-11-11T16:16:12,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/33ffdbc6f1c142b98ab56fcf9dc1aad9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/33ffdbc6f1c142b98ab56fcf9dc1aad9 2024-11-11T16:16:12,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/c37ba6bd73544695bf63a80a9873b9e7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/c37ba6bd73544695bf63a80a9873b9e7 2024-11-11T16:16:12,183 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/43f0453b7c1f48f08135cba5426a7b39 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/43f0453b7c1f48f08135cba5426a7b39 2024-11-11T16:16:12,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3201b186993c46d7bdf61e8c1506f212 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3201b186993c46d7bdf61e8c1506f212 2024-11-11T16:16:12,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/4a10e0ceef2c4b468d2bb92ac7f2be63 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/4a10e0ceef2c4b468d2bb92ac7f2be63 2024-11-11T16:16:12,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9077c4c321e84046b236d208243bc56d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9077c4c321e84046b236d208243bc56d 2024-11-11T16:16:12,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/29176ecba3414a04911b54d9b39e08b0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/29176ecba3414a04911b54d9b39e08b0 2024-11-11T16:16:12,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/e7346c0b53db4b3d9ce2c1d10acbc7ba to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/e7346c0b53db4b3d9ce2c1d10acbc7ba 2024-11-11T16:16:12,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/70ce831d62e4442e98bb315fedb954e7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/70ce831d62e4442e98bb315fedb954e7 2024-11-11T16:16:12,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/25dc7b494c5748069d3c2a3a9ae1bb74 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/25dc7b494c5748069d3c2a3a9ae1bb74 2024-11-11T16:16:12,189 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/2ba0229ae1ca44aeb54037c0ffef15eb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/2ba0229ae1ca44aeb54037c0ffef15eb 2024-11-11T16:16:12,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/82425fc9517747fe8a052dccdc48d133 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/82425fc9517747fe8a052dccdc48d133 2024-11-11T16:16:12,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/8b6827f814224299a29ef63f347ad22b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/8b6827f814224299a29ef63f347ad22b 2024-11-11T16:16:12,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/0969759805eb40e9966c669cc313eca6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/0969759805eb40e9966c669cc313eca6 2024-11-11T16:16:12,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/fc12b9bfd1384febb868a9b956bc4c77 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/fc12b9bfd1384febb868a9b956bc4c77 2024-11-11T16:16:12,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/135dc67f12a044d6a3b8161d1edf244f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/135dc67f12a044d6a3b8161d1edf244f 2024-11-11T16:16:12,194 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/808b04b83ae74b71838ccf56f3d6f8e7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/808b04b83ae74b71838ccf56f3d6f8e7 2024-11-11T16:16:12,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f9bc175587da480fabda1bb959c9d26e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/f9bc175587da480fabda1bb959c9d26e 2024-11-11T16:16:12,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/06cf190893464366ac21ba479033ccaa to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/06cf190893464366ac21ba479033ccaa 2024-11-11T16:16:12,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9c4ba0c9e803455ca3914cf5d8251a8a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/9c4ba0c9e803455ca3914cf5d8251a8a 2024-11-11T16:16:12,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/18ecb6ab7ed04360ae3fc2abfef508c4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/18ecb6ab7ed04360ae3fc2abfef508c4 2024-11-11T16:16:12,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/76996f2089f246a2ab9220faae808b79 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/76996f2089f246a2ab9220faae808b79 2024-11-11T16:16:12,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6bfe6cacf4fc4f819412643658efc4b2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6bfe6cacf4fc4f819412643658efc4b2 2024-11-11T16:16:12,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/02c87b175d36471abe4b10f032ced24d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/02c87b175d36471abe4b10f032ced24d 2024-11-11T16:16:12,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/094a5bbb438649eb97fc35e3f2f9855c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/094a5bbb438649eb97fc35e3f2f9855c 2024-11-11T16:16:12,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02ce31344caf4ccdb9a5a845ac9cb64b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e81258f861604a119fdfd75b41769cca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fd77a52c83a3422f9b6a48e8e21b0d5e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6b976bd80ee64024b433275d13eb6a4a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/5c58f7a13a244633a996291ad3678833, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0cff77898cda4c82b5787da2117ce00c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/cc315f99252a47eb9896158d94b1647d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/917c98cd3f664b5bbd2a010689234991, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c76f7cc61e8e44e7b183d887f4a5174e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/346522399fdf4543a36aa880ea4600b9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1547ed65dd404f4490e77f8e353c6f60, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/aff5293843fb4c9db78e78dc9f51b5e9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/d11cd12fff66432aae13d4ed32fd9d9e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fee582438eb142e0bc4ac2359e6e9cf3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1cae9f6d99984f2fb8cea0dfdcfdf631, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/90543d136b294c91a947bc9a977642df, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/defac8e66c734816be979d913e8f0be3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6be071df0e834d3c957f365efa09cdf1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e406ce37ed64425585e92a88c36a96e6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02725175963049fda7d254ef07658e03, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/9412e82ac60d443b857ffee927e109c6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0fc3d67683434664a9fe7624c6753fa1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/21b5cb99432d4154b7f55d0a17bff719, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/006f3bfefd80431db940dae4156eee01, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/34eef011b64c4feb80ccc3d1d68b1a43, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/be34c96e5cb04585a86d0da6860f3e3b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c745a88716e644f18e64295ce5105dd7] to archive 2024-11-11T16:16:12,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:16:12,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02ce31344caf4ccdb9a5a845ac9cb64b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02ce31344caf4ccdb9a5a845ac9cb64b 2024-11-11T16:16:12,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e81258f861604a119fdfd75b41769cca to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e81258f861604a119fdfd75b41769cca 2024-11-11T16:16:12,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fd77a52c83a3422f9b6a48e8e21b0d5e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fd77a52c83a3422f9b6a48e8e21b0d5e 2024-11-11T16:16:12,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6b976bd80ee64024b433275d13eb6a4a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6b976bd80ee64024b433275d13eb6a4a 2024-11-11T16:16:12,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/5c58f7a13a244633a996291ad3678833 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/5c58f7a13a244633a996291ad3678833 2024-11-11T16:16:12,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0cff77898cda4c82b5787da2117ce00c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0cff77898cda4c82b5787da2117ce00c 2024-11-11T16:16:12,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/cc315f99252a47eb9896158d94b1647d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/cc315f99252a47eb9896158d94b1647d 2024-11-11T16:16:12,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/917c98cd3f664b5bbd2a010689234991 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/917c98cd3f664b5bbd2a010689234991 2024-11-11T16:16:12,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c76f7cc61e8e44e7b183d887f4a5174e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c76f7cc61e8e44e7b183d887f4a5174e 2024-11-11T16:16:12,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/346522399fdf4543a36aa880ea4600b9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/346522399fdf4543a36aa880ea4600b9 2024-11-11T16:16:12,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1547ed65dd404f4490e77f8e353c6f60 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1547ed65dd404f4490e77f8e353c6f60 2024-11-11T16:16:12,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/aff5293843fb4c9db78e78dc9f51b5e9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/aff5293843fb4c9db78e78dc9f51b5e9 2024-11-11T16:16:12,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/d11cd12fff66432aae13d4ed32fd9d9e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/d11cd12fff66432aae13d4ed32fd9d9e 2024-11-11T16:16:12,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fee582438eb142e0bc4ac2359e6e9cf3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/fee582438eb142e0bc4ac2359e6e9cf3 2024-11-11T16:16:12,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1cae9f6d99984f2fb8cea0dfdcfdf631 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/1cae9f6d99984f2fb8cea0dfdcfdf631 2024-11-11T16:16:12,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/90543d136b294c91a947bc9a977642df to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/90543d136b294c91a947bc9a977642df 2024-11-11T16:16:12,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/defac8e66c734816be979d913e8f0be3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/defac8e66c734816be979d913e8f0be3 2024-11-11T16:16:12,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6be071df0e834d3c957f365efa09cdf1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/6be071df0e834d3c957f365efa09cdf1 2024-11-11T16:16:12,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e406ce37ed64425585e92a88c36a96e6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/e406ce37ed64425585e92a88c36a96e6 2024-11-11T16:16:12,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02725175963049fda7d254ef07658e03 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/02725175963049fda7d254ef07658e03 2024-11-11T16:16:12,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/9412e82ac60d443b857ffee927e109c6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/9412e82ac60d443b857ffee927e109c6 2024-11-11T16:16:12,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0fc3d67683434664a9fe7624c6753fa1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/0fc3d67683434664a9fe7624c6753fa1 2024-11-11T16:16:12,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/21b5cb99432d4154b7f55d0a17bff719 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/21b5cb99432d4154b7f55d0a17bff719 2024-11-11T16:16:12,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/006f3bfefd80431db940dae4156eee01 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/006f3bfefd80431db940dae4156eee01 2024-11-11T16:16:12,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/34eef011b64c4feb80ccc3d1d68b1a43 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/34eef011b64c4feb80ccc3d1d68b1a43 2024-11-11T16:16:12,230 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/be34c96e5cb04585a86d0da6860f3e3b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/be34c96e5cb04585a86d0da6860f3e3b 2024-11-11T16:16:12,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c745a88716e644f18e64295ce5105dd7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c745a88716e644f18e64295ce5105dd7 2024-11-11T16:16:12,318 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/2d5b58d9507640bda7f44e4bf96158ef 2024-11-11T16:16:12,322 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/A/8920466f96c046949cd630cccb65fcb8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8920466f96c046949cd630cccb65fcb8 2024-11-11T16:16:12,325 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8920466f96c046949cd630cccb65fcb8, entries=100, sequenceid=576, filesize=22.0 K 2024-11-11T16:16:12,325 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/B/3a094cc81aed482382ce15dea187cce0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3a094cc81aed482382ce15dea187cce0 2024-11-11T16:16:12,328 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3a094cc81aed482382ce15dea187cce0, entries=100, sequenceid=576, filesize=9.6 K 2024-11-11T16:16:12,329 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/.tmp/C/2d5b58d9507640bda7f44e4bf96158ef as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/2d5b58d9507640bda7f44e4bf96158ef 2024-11-11T16:16:12,332 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/2d5b58d9507640bda7f44e4bf96158ef, entries=100, sequenceid=576, filesize=9.6 K 2024-11-11T16:16:12,333 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 79645af1f242c463cd05cc60d56c4bdc in 1653ms, sequenceid=576, compaction requested=true 2024-11-11T16:16:12,336 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/recovered.edits/579.seqid, newMaxSeqId=579, maxSeqId=4 2024-11-11T16:16:12,337 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. 2024-11-11T16:16:12,337 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 79645af1f242c463cd05cc60d56c4bdc: Waiting for close lock at 1731341770679Running coprocessor pre-close hooks at 1731341770679Disabling compacts and flushes for region at 1731341770679Disabling writes for close at 1731341770679Obtaining lock to block concurrent updates at 1731341770680 (+1 ms)Preparing flush snapshotting stores in 79645af1f242c463cd05cc60d56c4bdc at 1731341770680Finished memstore snapshotting TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731341770680Flushing stores of TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc. at 1731341770681 (+1 ms)Flushing 79645af1f242c463cd05cc60d56c4bdc/A: creating writer at 1731341770681Flushing mob file 79645af1f242c463cd05cc60d56c4bdc/A: appending metadata at 1731341770686 (+5 ms)Flushing mob file 79645af1f242c463cd05cc60d56c4bdc/A: closing flushed file at 1731341770686Flushing 79645af1f242c463cd05cc60d56c4bdc/A: appending metadata at 1731341771094 (+408 ms)Flushing 79645af1f242c463cd05cc60d56c4bdc/A: closing flushed file at 1731341771094Flushing 79645af1f242c463cd05cc60d56c4bdc/B: creating writer at 1731341771502 (+408 ms)Flushing 79645af1f242c463cd05cc60d56c4bdc/B: appending metadata at 1731341771504 (+2 ms)Flushing 79645af1f242c463cd05cc60d56c4bdc/B: closing flushed file at 1731341771504Flushing 79645af1f242c463cd05cc60d56c4bdc/C: creating writer at 1731341771912 (+408 ms)Flushing 79645af1f242c463cd05cc60d56c4bdc/C: appending metadata at 1731341771914 (+2 ms)Flushing 79645af1f242c463cd05cc60d56c4bdc/C: closing flushed file at 1731341771914Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5be59480: reopening flushed file at 1731341772321 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@699e1f58: reopening flushed file at 1731341772325 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27e4d7bf: reopening flushed file at 1731341772328 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 79645af1f242c463cd05cc60d56c4bdc in 1653ms, sequenceid=576, compaction requested=true at 1731341772333 (+5 ms)Writing region close event to WAL at 1731341772334 (+1 ms)Running coprocessor post-close hooks at 1731341772336 (+2 ms)Closed at 1731341772337 (+1 ms) 2024-11-11T16:16:12,338 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,339 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=79645af1f242c463cd05cc60d56c4bdc, regionState=CLOSED 2024-11-11T16:16:12,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:16:12,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T16:16:12,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 79645af1f242c463cd05cc60d56c4bdc, server=16b413a53992,45121,1731341648557 in 1.8130 sec 2024-11-11T16:16:12,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-11T16:16:12,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=79645af1f242c463cd05cc60d56c4bdc, UNASSIGN in 1.8180 sec 2024-11-11T16:16:12,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T16:16:12,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8210 sec 2024-11-11T16:16:12,346 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341772346"}]},"ts":"1731341772346"} 2024-11-11T16:16:12,347 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T16:16:12,347 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T16:16:12,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8300 sec 2024-11-11T16:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T16:16:12,666 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:12,666 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T16:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:12,668 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T16:16:12,669 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:12,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,673 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/recovered.edits] 2024-11-11T16:16:12,675 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/30150e4834b34a20ad4e1f693732cc14 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/30150e4834b34a20ad4e1f693732cc14 2024-11-11T16:16:12,676 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8920466f96c046949cd630cccb65fcb8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/8920466f96c046949cd630cccb65fcb8 2024-11-11T16:16:12,677 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b5e849b547cf44048e8bc5f1e22cb1dc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/A/b5e849b547cf44048e8bc5f1e22cb1dc 2024-11-11T16:16:12,679 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3a094cc81aed482382ce15dea187cce0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/3a094cc81aed482382ce15dea187cce0 2024-11-11T16:16:12,680 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/5aad01123a2b4e57a4cc9f37aee0675b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/5aad01123a2b4e57a4cc9f37aee0675b 2024-11-11T16:16:12,681 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6411ab3d58744004b42a7de8b11d8c8a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/B/6411ab3d58744004b42a7de8b11d8c8a 2024-11-11T16:16:12,683 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/2d5b58d9507640bda7f44e4bf96158ef to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/2d5b58d9507640bda7f44e4bf96158ef 2024-11-11T16:16:12,684 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/610c85266beb428c9231e884428aa9f0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/610c85266beb428c9231e884428aa9f0 2024-11-11T16:16:12,685 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c9ee4c39f0b74ac7a3cf63dd18a6abc2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/C/c9ee4c39f0b74ac7a3cf63dd18a6abc2 2024-11-11T16:16:12,687 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/recovered.edits/579.seqid to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc/recovered.edits/579.seqid 2024-11-11T16:16:12,687 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,687 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T16:16:12,688 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T16:16:12,688 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T16:16:12,691 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106abd73f4896429abd2e292f54b748a7_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106abd73f4896429abd2e292f54b748a7_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,692 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117b40560a3b440cd80d7a8d211511c3f_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117b40560a3b440cd80d7a8d211511c3f_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,693 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111bfee089941e41438aa3e0d7cec75207_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111bfee089941e41438aa3e0d7cec75207_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,694 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112272e2d80cf346e7a7ec1a2dfb87f17d_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112272e2d80cf346e7a7ec1a2dfb87f17d_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,695 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111245173e4eec2406bb5bd57b77c85e514_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111245173e4eec2406bb5bd57b77c85e514_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,696 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112c4a675690fd4205aadb9015f5646885_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112c4a675690fd4205aadb9015f5646885_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,697 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111135183f5cdbaa42ddbed897963d6956d6_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111135183f5cdbaa42ddbed897963d6956d6_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,698 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113535aeac14a84ad0b4b23fe0fdbb99b3_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113535aeac14a84ad0b4b23fe0fdbb99b3_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,699 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149e9a0e0ff654116a6bb80552360c9f0_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149e9a0e0ff654116a6bb80552360c9f0_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,700 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111153eab99d45db40a68d03931be55397b0_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111153eab99d45db40a68d03931be55397b0_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,701 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d7dc4ea08494178b9eaea9d352313dc_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d7dc4ea08494178b9eaea9d352313dc_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,702 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111652bd80c633446e5b72c835ba57624e8_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111652bd80c633446e5b72c835ba57624e8_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,703 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116cfc4ac249964852a91dfef0e92f1375_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116cfc4ac249964852a91dfef0e92f1375_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,703 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118ade6da78db24a85a6282daec2ad96d5_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118ade6da78db24a85a6282daec2ad96d5_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,704 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118f6222d17eea42daac0a75fff4fb0b6d_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118f6222d17eea42daac0a75fff4fb0b6d_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,705 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b77ef60a529645059abdc6fa05551fe8_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b77ef60a529645059abdc6fa05551fe8_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,706 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ba6c0e51318c4509bd6102537703c134_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ba6c0e51318c4509bd6102537703c134_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,707 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c42cb05591bf43f2ac4aec986e2521e1_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c42cb05591bf43f2ac4aec986e2521e1_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,708 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3bcfd5363964bab85742f5a24e0c55f_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d3bcfd5363964bab85742f5a24e0c55f_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,709 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111de2c3cc279324d819fff4612f7eb1235_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111de2c3cc279324d819fff4612f7eb1235_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,710 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f1bb367e8b4044c7aa7a70d5a05c3e74_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f1bb367e8b4044c7aa7a70d5a05c3e74_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,711 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ffb7bc4fb9964fbb80c1265743f1e0f7_79645af1f242c463cd05cc60d56c4bdc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ffb7bc4fb9964fbb80c1265743f1e0f7_79645af1f242c463cd05cc60d56c4bdc 2024-11-11T16:16:12,711 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T16:16:12,713 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:12,715 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T16:16:12,717 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T16:16:12,718 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:12,718 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T16:16:12,718 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731341772718"}]},"ts":"9223372036854775807"} 2024-11-11T16:16:12,720 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T16:16:12,720 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 79645af1f242c463cd05cc60d56c4bdc, NAME => 'TestAcidGuarantees,,1731341743616.79645af1f242c463cd05cc60d56c4bdc.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T16:16:12,720 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T16:16:12,720 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731341772720"}]},"ts":"9223372036854775807"} 2024-11-11T16:16:12,721 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T16:16:12,722 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:12,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-11T16:16:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T16:16:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T16:16:12,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:12,786 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1554041885_22 at /127.0.0.1:36710 [Waiting for operation #812] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1306082440_22 at /127.0.0.1:36530 [Waiting for operation #917] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/16b413a53992:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1554041885_22 at /127.0.0.1:50162 [Waiting for operation #997] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1306082440_22 at /127.0.0.1:36698 [Waiting for operation #876] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=600 (was 665), ProcessCount=11 (was 11), AvailableMemoryMB=303 (was 1420) 2024-11-11T16:16:12,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=600, ProcessCount=11, AvailableMemoryMB=302 2024-11-11T16:16:12,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:16:12,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:12,798 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T16:16:12,798 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:12,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-11T16:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-11T16:16:12,798 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T16:16:12,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742288_1464 (size=1061) 2024-11-11T16:16:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-11T16:16:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-11T16:16:13,205 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:16:13,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742289_1465 (size=53) 2024-11-11T16:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-11T16:16:13,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:16:13,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9e768d9f517a9b5cf0bf527b36f5956b, disabling compactions & flushes 2024-11-11T16:16:13,610 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. after waiting 0 ms 2024-11-11T16:16:13,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,610 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,610 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e768d9f517a9b5cf0bf527b36f5956b: Waiting for close lock at 1731341773610Disabling compacts and flushes for region at 1731341773610Disabling writes for close at 1731341773610Writing region close event to WAL at 1731341773610Closed at 1731341773610 2024-11-11T16:16:13,612 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T16:16:13,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731341773612"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341773612"}]},"ts":"1731341773612"} 2024-11-11T16:16:13,614 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T16:16:13,614 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T16:16:13,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341773614"}]},"ts":"1731341773614"} 2024-11-11T16:16:13,615 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T16:16:13,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, ASSIGN}] 2024-11-11T16:16:13,616 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, ASSIGN 2024-11-11T16:16:13,617 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, ASSIGN; state=OFFLINE, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=false 2024-11-11T16:16:13,767 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9e768d9f517a9b5cf0bf527b36f5956b, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:13,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, ASSIGN because future has completed 2024-11-11T16:16:13,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:16:13,923 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,923 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:16:13,924 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,924 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:16:13,924 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,924 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,925 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,926 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:13,926 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e768d9f517a9b5cf0bf527b36f5956b columnFamilyName A 2024-11-11T16:16:13,927 DEBUG [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:13,927 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.HStore(327): Store=9e768d9f517a9b5cf0bf527b36f5956b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:13,927 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,928 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:13,928 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e768d9f517a9b5cf0bf527b36f5956b columnFamilyName B 2024-11-11T16:16:13,928 DEBUG [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:13,928 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.HStore(327): Store=9e768d9f517a9b5cf0bf527b36f5956b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:13,929 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,929 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:13,929 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e768d9f517a9b5cf0bf527b36f5956b columnFamilyName C 2024-11-11T16:16:13,929 DEBUG [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:13,930 INFO [StoreOpener-9e768d9f517a9b5cf0bf527b36f5956b-1 {}] regionserver.HStore(327): Store=9e768d9f517a9b5cf0bf527b36f5956b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:13,930 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,930 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,930 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,931 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,931 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,931 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,932 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:16:13,932 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,934 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:16:13,934 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 9e768d9f517a9b5cf0bf527b36f5956b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67599508, jitterRate=0.0073111653327941895}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:16:13,934 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:13,935 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 9e768d9f517a9b5cf0bf527b36f5956b: Running coprocessor pre-open hook at 1731341773924Writing region info on filesystem at 1731341773924Initializing all the Stores at 1731341773925 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341773925Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341773925Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341773925Cleaning up temporary data from old regions at 1731341773931 (+6 ms)Running coprocessor post-open hooks at 1731341773934 (+3 ms)Region opened successfully at 1731341773935 (+1 ms) 2024-11-11T16:16:13,936 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., pid=114, masterSystemTime=1731341773921 2024-11-11T16:16:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-11T16:16:13,937 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,937 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:13,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9e768d9f517a9b5cf0bf527b36f5956b, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:13,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:16:13,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T16:16:13,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 in 171 msec 2024-11-11T16:16:13,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-11T16:16:13,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, ASSIGN in 325 msec 2024-11-11T16:16:13,943 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T16:16:13,943 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341773943"}]},"ts":"1731341773943"} 2024-11-11T16:16:13,944 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T16:16:13,945 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T16:16:13,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-11T16:16:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-11T16:16:14,946 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:14,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f33d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,948 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f99e46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,950 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@630305f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,951 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,953 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,33601,1731341647570 2024-11-11T16:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7138cbd0 2024-11-11T16:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:16:14,954 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:16:14,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab365a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,956 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7434c0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,957 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ffd695f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31de1c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,959 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35bc7cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,960 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7e37c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95f8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,961 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a37a8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,963 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254fbd0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de313b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,964 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2b1fe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@386186dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb00bf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,967 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a37173b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,968 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b7e203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2c2c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,971 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec64eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,972 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ba54b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3b7179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,974 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197b1c4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,975 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@352feff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3037f761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,977 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a665e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,978 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58264016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300eb3da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c548aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,983 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405abb3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a03b32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:14,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:14,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cf020a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:14,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56834426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:14,986 DEBUG [Thread-2088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,987 DEBUG [Thread-2090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,987 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:14,987 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,987 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,987 DEBUG [Thread-2083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T16:16:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,988 DEBUG [Thread-2077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T16:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,990 DEBUG [Thread-2081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,990 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,992 DEBUG [Thread-2081 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,992 DEBUG [Thread-2081 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,995 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:14,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:14,996 DEBUG [Thread-2092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:14,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:15,003 DEBUG [Thread-2079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:15,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:16:15,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:15,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:15,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:15,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:15,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:15,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:15,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2] 2024-11-11T16:16:15,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/301bdf57e33f43e085ee10e76f230172 is 50, key is test_row_1/A:col10/1731341775003/Put/seqid=0 2024-11-11T16:16:15,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742290_1466 (size=9657) 2024-11-11T16:16:15,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341835041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341775032, max timestamp=1731341775042Num uniques 859; 2024-11-11T16:16:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:15,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341775032, max timestamp=1731341775042Num uniques 859; 2024-11-11T16:16:15,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:15,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341775032, max timestamp=1731341775042Num uniques 859; 2024-11-11T16:16:15,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341835082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341835083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341835086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T16:16:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341835088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341835156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341835205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341835205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341835205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341835205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T16:16:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341835376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341835426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341835426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341835426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341835426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/301bdf57e33f43e085ee10e76f230172 2024-11-11T16:16:15,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/b6a08f0af9ea417694f48e0f730abae4 is 50, key is test_row_1/B:col10/1731341775003/Put/seqid=0 2024-11-11T16:16:15,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742291_1467 (size=9657) 2024-11-11T16:16:15,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T16:16:15,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341835696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341835746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341835746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341835746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341835746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:15,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/b6a08f0af9ea417694f48e0f730abae4 2024-11-11T16:16:15,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/664adad8747d44e7b9402c230ff7725e is 50, key is test_row_1/C:col10/1731341775003/Put/seqid=0 2024-11-11T16:16:15,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742292_1468 (size=9657) 2024-11-11T16:16:15,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:15,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:16,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:16,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T16:16:16,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:16,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:16,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:16,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:16,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:16,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341836215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341836266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341836266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341836266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341836266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:16,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/664adad8747d44e7b9402c230ff7725e 2024-11-11T16:16:16,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/301bdf57e33f43e085ee10e76f230172 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/301bdf57e33f43e085ee10e76f230172 2024-11-11T16:16:16,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/301bdf57e33f43e085ee10e76f230172, entries=100, sequenceid=11, filesize=9.4 K 2024-11-11T16:16:16,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/b6a08f0af9ea417694f48e0f730abae4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b6a08f0af9ea417694f48e0f730abae4 2024-11-11T16:16:16,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b6a08f0af9ea417694f48e0f730abae4, entries=100, sequenceid=11, filesize=9.4 K 2024-11-11T16:16:16,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/664adad8747d44e7b9402c230ff7725e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/664adad8747d44e7b9402c230ff7725e 2024-11-11T16:16:16,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/664adad8747d44e7b9402c230ff7725e, entries=100, sequenceid=11, filesize=9.4 K 2024-11-11T16:16:16,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9e768d9f517a9b5cf0bf527b36f5956b in 1305ms, sequenceid=11, compaction requested=false 2024-11-11T16:16:16,311 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T16:16:16,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:16,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T16:16:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:16,375 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T16:16:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:16,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/64213a7b7a1640738d09a5f672ed3330 is 50, key is test_row_0/A:col10/1731341775084/Put/seqid=0 2024-11-11T16:16:16,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742293_1469 (size=12001) 2024-11-11T16:16:16,384 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/64213a7b7a1640738d09a5f672ed3330 2024-11-11T16:16:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e1be90577dc40f4b5ad3a15f1335e79 is 50, key is test_row_0/B:col10/1731341775084/Put/seqid=0 2024-11-11T16:16:16,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742294_1470 (size=12001) 2024-11-11T16:16:16,798 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e1be90577dc40f4b5ad3a15f1335e79 2024-11-11T16:16:16,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/8bb547c48b4a423eb205d2414dc393c0 is 50, key is test_row_0/C:col10/1731341775084/Put/seqid=0 2024-11-11T16:16:16,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742295_1471 (size=12001) 2024-11-11T16:16:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T16:16:17,211 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/8bb547c48b4a423eb205d2414dc393c0 2024-11-11T16:16:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/64213a7b7a1640738d09a5f672ed3330 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/64213a7b7a1640738d09a5f672ed3330 2024-11-11T16:16:17,219 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/64213a7b7a1640738d09a5f672ed3330, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T16:16:17,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e1be90577dc40f4b5ad3a15f1335e79 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e1be90577dc40f4b5ad3a15f1335e79 2024-11-11T16:16:17,223 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e1be90577dc40f4b5ad3a15f1335e79, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T16:16:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/8bb547c48b4a423eb205d2414dc393c0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/8bb547c48b4a423eb205d2414dc393c0 2024-11-11T16:16:17,228 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/8bb547c48b4a423eb205d2414dc393c0, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T16:16:17,228 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 9e768d9f517a9b5cf0bf527b36f5956b in 853ms, sequenceid=48, compaction requested=false 2024-11-11T16:16:17,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:17,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:17,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T16:16:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T16:16:17,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T16:16:17,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2350 sec 2024-11-11T16:16:17,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.2450 sec 2024-11-11T16:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:17,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:16:17,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:17,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:17,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:17,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:17,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:17,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:17,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/03d9d4332f734ebda612eb1010c73fd8 is 50, key is test_row_0/A:col10/1731341777248/Put/seqid=0 2024-11-11T16:16:17,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742296_1472 (size=12001) 2024-11-11T16:16:17,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/03d9d4332f734ebda612eb1010c73fd8 2024-11-11T16:16:17,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/3f2bdf0025ff4a3d8f82891be2d2ebc1 is 50, key is test_row_0/B:col10/1731341777248/Put/seqid=0 2024-11-11T16:16:17,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742297_1473 (size=12001) 2024-11-11T16:16:17,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/3f2bdf0025ff4a3d8f82891be2d2ebc1 2024-11-11T16:16:17,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/264687af98b1443fbcb408338a11eaf7 is 50, key is test_row_0/C:col10/1731341777248/Put/seqid=0 2024-11-11T16:16:17,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742298_1474 (size=12001) 2024-11-11T16:16:17,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341777295, max timestamp=1731341777313Num uniques 859; 2024-11-11T16:16:17,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:17,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341777295, max timestamp=1731341777313Num uniques 859; 2024-11-11T16:16:17,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341777295, max timestamp=1731341777313Num uniques 859; 2024-11-11T16:16:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341837347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341837356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341837358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341837364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341837365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341837476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341837486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341837486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341837487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341837487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/264687af98b1443fbcb408338a11eaf7 2024-11-11T16:16:17,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341837696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/03d9d4332f734ebda612eb1010c73fd8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/03d9d4332f734ebda612eb1010c73fd8 2024-11-11T16:16:17,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/03d9d4332f734ebda612eb1010c73fd8, entries=150, sequenceid=58, filesize=11.7 K 2024-11-11T16:16:17,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/3f2bdf0025ff4a3d8f82891be2d2ebc1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3f2bdf0025ff4a3d8f82891be2d2ebc1 2024-11-11T16:16:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341837706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3f2bdf0025ff4a3d8f82891be2d2ebc1, entries=150, sequenceid=58, filesize=11.7 K 2024-11-11T16:16:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341837706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341837706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/264687af98b1443fbcb408338a11eaf7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/264687af98b1443fbcb408338a11eaf7 2024-11-11T16:16:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341837709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:17,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/264687af98b1443fbcb408338a11eaf7, entries=150, sequenceid=58, filesize=11.7 K 2024-11-11T16:16:17,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 9e768d9f517a9b5cf0bf527b36f5956b in 477ms, sequenceid=58, compaction requested=true 2024-11-11T16:16:17,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:17,729 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:17,730 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:17,733 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:17,733 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:17,733 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:17,733 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/301bdf57e33f43e085ee10e76f230172, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/64213a7b7a1640738d09a5f672ed3330, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/03d9d4332f734ebda612eb1010c73fd8] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=32.9 K 2024-11-11T16:16:17,735 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:17,735 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:17,735 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:17,735 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b6a08f0af9ea417694f48e0f730abae4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e1be90577dc40f4b5ad3a15f1335e79, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3f2bdf0025ff4a3d8f82891be2d2ebc1] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=32.9 K 2024-11-11T16:16:17,735 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301bdf57e33f43e085ee10e76f230172, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731341774997 2024-11-11T16:16:17,736 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b6a08f0af9ea417694f48e0f730abae4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731341774997 2024-11-11T16:16:17,736 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64213a7b7a1640738d09a5f672ed3330, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341775069 2024-11-11T16:16:17,736 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1be90577dc40f4b5ad3a15f1335e79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341775069 2024-11-11T16:16:17,736 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03d9d4332f734ebda612eb1010c73fd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731341777245 2024-11-11T16:16:17,737 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f2bdf0025ff4a3d8f82891be2d2ebc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731341777245 2024-11-11T16:16:17,758 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:17,759 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/d66877a739524e6cb41bbc4e9a2a301b is 50, key is test_row_0/B:col10/1731341777248/Put/seqid=0 2024-11-11T16:16:17,767 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:17,767 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/eea469229b3d480db89f9976ea561378 is 50, key is test_row_0/A:col10/1731341777248/Put/seqid=0 2024-11-11T16:16:17,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742299_1475 (size=12104) 2024-11-11T16:16:17,832 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/d66877a739524e6cb41bbc4e9a2a301b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/d66877a739524e6cb41bbc4e9a2a301b 2024-11-11T16:16:17,837 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into d66877a739524e6cb41bbc4e9a2a301b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:17,837 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:17,838 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=13, startTime=1731341777730; duration=0sec 2024-11-11T16:16:17,838 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:17,838 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:17,838 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:17,839 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:17,839 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:17,839 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:17,839 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/664adad8747d44e7b9402c230ff7725e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/8bb547c48b4a423eb205d2414dc393c0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/264687af98b1443fbcb408338a11eaf7] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=32.9 K 2024-11-11T16:16:17,839 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 664adad8747d44e7b9402c230ff7725e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731341774997 2024-11-11T16:16:17,840 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bb547c48b4a423eb205d2414dc393c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731341775069 2024-11-11T16:16:17,840 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 264687af98b1443fbcb408338a11eaf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731341777245 2024-11-11T16:16:17,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742300_1476 (size=12104) 2024-11-11T16:16:17,856 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T16:16:17,863 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/eea469229b3d480db89f9976ea561378 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eea469229b3d480db89f9976ea561378 2024-11-11T16:16:17,875 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into eea469229b3d480db89f9976ea561378(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:17,875 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:17,875 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=13, startTime=1731341777729; duration=0sec 2024-11-11T16:16:17,875 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:17,875 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:17,892 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:17,893 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/23567a779c424b578cf1d0ce781da5bf is 50, key is test_row_0/C:col10/1731341777248/Put/seqid=0 2024-11-11T16:16:17,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742301_1477 (size=12104) 2024-11-11T16:16:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:18,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-11T16:16:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:18,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b65d63904c4945c89d1adf88b22aedfe is 50, key is test_row_0/A:col10/1731341777364/Put/seqid=0 2024-11-11T16:16:18,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742302_1478 (size=14341) 2024-11-11T16:16:18,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b65d63904c4945c89d1adf88b22aedfe 2024-11-11T16:16:18,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/882c490b24da4be1ba6cb71175e1c1e5 is 50, key is test_row_0/B:col10/1731341777364/Put/seqid=0 2024-11-11T16:16:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341838040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341838041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341838042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341838043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341838055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742303_1479 (size=12001) 2024-11-11T16:16:18,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/882c490b24da4be1ba6cb71175e1c1e5 2024-11-11T16:16:18,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/01d8fad6ef904a8ba8248c398ddcb267 is 50, key is test_row_0/C:col10/1731341777364/Put/seqid=0 2024-11-11T16:16:18,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742304_1480 (size=12001) 2024-11-11T16:16:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341838166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341838166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341838166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341838166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341838167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,331 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/23567a779c424b578cf1d0ce781da5bf as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/23567a779c424b578cf1d0ce781da5bf 2024-11-11T16:16:18,335 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 23567a779c424b578cf1d0ce781da5bf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:18,335 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:18,335 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=13, startTime=1731341777730; duration=0sec 2024-11-11T16:16:18,335 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:18,336 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341838375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341838386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341838387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341838387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341838387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/01d8fad6ef904a8ba8248c398ddcb267 2024-11-11T16:16:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b65d63904c4945c89d1adf88b22aedfe as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b65d63904c4945c89d1adf88b22aedfe 2024-11-11T16:16:18,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b65d63904c4945c89d1adf88b22aedfe, entries=200, sequenceid=96, filesize=14.0 K 2024-11-11T16:16:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/882c490b24da4be1ba6cb71175e1c1e5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/882c490b24da4be1ba6cb71175e1c1e5 2024-11-11T16:16:18,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/882c490b24da4be1ba6cb71175e1c1e5, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T16:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/01d8fad6ef904a8ba8248c398ddcb267 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/01d8fad6ef904a8ba8248c398ddcb267 2024-11-11T16:16:18,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/01d8fad6ef904a8ba8248c398ddcb267, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T16:16:18,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 9e768d9f517a9b5cf0bf527b36f5956b in 477ms, sequenceid=96, compaction requested=false 2024-11-11T16:16:18,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:18,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:16:18,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:18,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:18,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:18,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:18,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:18,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:18,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f59f06a332454bd5a99e9288c4f0a849 is 50, key is test_row_0/A:col10/1731341778698/Put/seqid=0 2024-11-11T16:16:18,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742305_1481 (size=12001) 2024-11-11T16:16:18,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341778726, max timestamp=1731341778745Num uniques 859; 2024-11-11T16:16:18,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:18,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341778726, max timestamp=1731341778745Num uniques 859; 2024-11-11T16:16:18,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:18,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341778726, max timestamp=1731341778745Num uniques 859; 2024-11-11T16:16:18,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341838775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341838777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341838778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341838780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341838792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341838895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341838895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341838896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341838906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341838906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:18,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f59f06a332454bd5a99e9288c4f0a849 2024-11-11T16:16:19,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341839106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341839106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341839106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ac9ec8d1a8e7410fb1e9f3ea443337d1 is 50, key is test_row_0/B:col10/1731341778698/Put/seqid=0 2024-11-11T16:16:19,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742306_1482 (size=12001) 2024-11-11T16:16:19,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ac9ec8d1a8e7410fb1e9f3ea443337d1 2024-11-11T16:16:19,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/ee5137b2936c46658a08512d749a502b is 50, key is test_row_0/C:col10/1731341778698/Put/seqid=0 2024-11-11T16:16:19,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341839125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341839125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742307_1483 (size=12001) 2024-11-11T16:16:19,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/ee5137b2936c46658a08512d749a502b 2024-11-11T16:16:19,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f59f06a332454bd5a99e9288c4f0a849 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f59f06a332454bd5a99e9288c4f0a849 2024-11-11T16:16:19,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f59f06a332454bd5a99e9288c4f0a849, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T16:16:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T16:16:19,156 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:19,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T16:16:19,159 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T16:16:19,160 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:19,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:19,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ac9ec8d1a8e7410fb1e9f3ea443337d1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac9ec8d1a8e7410fb1e9f3ea443337d1 2024-11-11T16:16:19,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac9ec8d1a8e7410fb1e9f3ea443337d1, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T16:16:19,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/ee5137b2936c46658a08512d749a502b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/ee5137b2936c46658a08512d749a502b 2024-11-11T16:16:19,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/ee5137b2936c46658a08512d749a502b, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T16:16:19,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 9e768d9f517a9b5cf0bf527b36f5956b in 472ms, sequenceid=107, compaction requested=true 2024-11-11T16:16:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:19,172 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:16:19,172 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:19,173 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:19,173 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:19,173 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,173 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eea469229b3d480db89f9976ea561378, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b65d63904c4945c89d1adf88b22aedfe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f59f06a332454bd5a99e9288c4f0a849] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=37.5 K 2024-11-11T16:16:19,174 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:19,174 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:19,174 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,174 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/d66877a739524e6cb41bbc4e9a2a301b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/882c490b24da4be1ba6cb71175e1c1e5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac9ec8d1a8e7410fb1e9f3ea443337d1] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=35.3 K 2024-11-11T16:16:19,174 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea469229b3d480db89f9976ea561378, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731341777245 2024-11-11T16:16:19,174 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d66877a739524e6cb41bbc4e9a2a301b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731341777245 2024-11-11T16:16:19,174 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b65d63904c4945c89d1adf88b22aedfe, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731341777353 2024-11-11T16:16:19,175 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 882c490b24da4be1ba6cb71175e1c1e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731341777362 2024-11-11T16:16:19,175 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59f06a332454bd5a99e9288c4f0a849, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731341778041 2024-11-11T16:16:19,175 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9ec8d1a8e7410fb1e9f3ea443337d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731341778041 2024-11-11T16:16:19,191 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:19,191 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:19,191 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/5ae5390c930d43ba810d4a387addbb48 is 50, key is test_row_0/B:col10/1731341778698/Put/seqid=0 2024-11-11T16:16:19,191 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/42405786a66440ae91aa79d51a6f9b63 is 50, key is test_row_0/A:col10/1731341778698/Put/seqid=0 2024-11-11T16:16:19,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742308_1484 (size=12207) 2024-11-11T16:16:19,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742309_1485 (size=12207) 2024-11-11T16:16:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T16:16:19,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T16:16:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,313 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-11T16:16:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:19,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:19,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/48dc1dbcf4004914bd04a238d411572c is 50, key is test_row_0/A:col10/1731341778790/Put/seqid=0 2024-11-11T16:16:19,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742310_1486 (size=12151) 2024-11-11T16:16:19,329 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/48dc1dbcf4004914bd04a238d411572c 2024-11-11T16:16:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/37baae1348c441fd9bdadc5b9fd04d29 is 50, key is test_row_0/B:col10/1731341778790/Put/seqid=0 2024-11-11T16:16:19,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742311_1487 (size=12151) 2024-11-11T16:16:19,344 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/37baae1348c441fd9bdadc5b9fd04d29 2024-11-11T16:16:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/d51d85d60c564fd5aef8b636814f4f6b is 50, key is test_row_0/C:col10/1731341778790/Put/seqid=0 2024-11-11T16:16:19,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742312_1488 (size=12151) 2024-11-11T16:16:19,369 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/d51d85d60c564fd5aef8b636814f4f6b 2024-11-11T16:16:19,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/48dc1dbcf4004914bd04a238d411572c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/48dc1dbcf4004914bd04a238d411572c 2024-11-11T16:16:19,386 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/48dc1dbcf4004914bd04a238d411572c, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T16:16:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/37baae1348c441fd9bdadc5b9fd04d29 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/37baae1348c441fd9bdadc5b9fd04d29 2024-11-11T16:16:19,392 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/37baae1348c441fd9bdadc5b9fd04d29, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T16:16:19,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/d51d85d60c564fd5aef8b636814f4f6b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d51d85d60c564fd5aef8b636814f4f6b 2024-11-11T16:16:19,397 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d51d85d60c564fd5aef8b636814f4f6b, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T16:16:19,403 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=0 B/0 for 9e768d9f517a9b5cf0bf527b36f5956b in 90ms, sequenceid=142, compaction requested=true 2024-11-11T16:16:19,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:19,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T16:16:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T16:16:19,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T16:16:19,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 245 msec 2024-11-11T16:16:19,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 250 msec 2024-11-11T16:16:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:19,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:16:19,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:19,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:19,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:19,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/eaa64707151e4ea6b6d809b7d9447571 is 50, key is test_row_0/A:col10/1731341779435/Put/seqid=0 2024-11-11T16:16:19,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742313_1489 (size=16931) 2024-11-11T16:16:19,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/eaa64707151e4ea6b6d809b7d9447571 2024-11-11T16:16:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T16:16:19,477 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:19,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:19,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/c9fdfa88326646c6a945cae45ed9953e is 50, key is test_row_0/B:col10/1731341779435/Put/seqid=0 2024-11-11T16:16:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T16:16:19,483 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T16:16:19,484 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:19,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:19,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742314_1490 (size=12151) 2024-11-11T16:16:19,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341779496, max timestamp=1731341779513Num uniques 859; 2024-11-11T16:16:19,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341779496, max timestamp=1731341779513Num uniques 859; 2024-11-11T16:16:19,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341779496, max timestamp=1731341779513Num uniques 859; 2024-11-11T16:16:19,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:19,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:19,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:19,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341839563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341839563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T16:16:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341839567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341839569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341839572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,614 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/5ae5390c930d43ba810d4a387addbb48 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5ae5390c930d43ba810d4a387addbb48 2024-11-11T16:16:19,621 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into 5ae5390c930d43ba810d4a387addbb48(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:19,621 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:19,621 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=13, startTime=1731341779172; duration=0sec 2024-11-11T16:16:19,621 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:19,621 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:19,621 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:19,624 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:19,624 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:19,624 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,624 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/23567a779c424b578cf1d0ce781da5bf, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/01d8fad6ef904a8ba8248c398ddcb267, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/ee5137b2936c46658a08512d749a502b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d51d85d60c564fd5aef8b636814f4f6b] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=47.1 K 2024-11-11T16:16:19,624 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/42405786a66440ae91aa79d51a6f9b63 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/42405786a66440ae91aa79d51a6f9b63 2024-11-11T16:16:19,625 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 23567a779c424b578cf1d0ce781da5bf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731341777245 2024-11-11T16:16:19,625 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 01d8fad6ef904a8ba8248c398ddcb267, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731341777362 2024-11-11T16:16:19,625 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5137b2936c46658a08512d749a502b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731341778041 2024-11-11T16:16:19,625 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d51d85d60c564fd5aef8b636814f4f6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731341778775 2024-11-11T16:16:19,629 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into 42405786a66440ae91aa79d51a6f9b63(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:19,629 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:19,629 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=13, startTime=1731341779172; duration=0sec 2024-11-11T16:16:19,629 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:19,629 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:19,635 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:19,636 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9c8f8fd841764733bbf74f5ccebbd20b is 50, key is test_row_0/C:col10/1731341778790/Put/seqid=0 2024-11-11T16:16:19,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:19,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:19,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742315_1491 (size=12391) 2024-11-11T16:16:19,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341839686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341839686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341839695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341839695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341839695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:19,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:19,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T16:16:19,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/c9fdfa88326646c6a945cae45ed9953e 2024-11-11T16:16:19,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/782002e66b844a03a203c3b705ac3764 is 50, key is test_row_0/C:col10/1731341779435/Put/seqid=0 2024-11-11T16:16:19,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742316_1492 (size=12151) 2024-11-11T16:16:19,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/782002e66b844a03a203c3b705ac3764 2024-11-11T16:16:19,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/eaa64707151e4ea6b6d809b7d9447571 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eaa64707151e4ea6b6d809b7d9447571 2024-11-11T16:16:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341839906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341839906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eaa64707151e4ea6b6d809b7d9447571, entries=250, sequenceid=152, filesize=16.5 K 2024-11-11T16:16:19,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/c9fdfa88326646c6a945cae45ed9953e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c9fdfa88326646c6a945cae45ed9953e 2024-11-11T16:16:19,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c9fdfa88326646c6a945cae45ed9953e, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T16:16:19,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/782002e66b844a03a203c3b705ac3764 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/782002e66b844a03a203c3b705ac3764 2024-11-11T16:16:19,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/782002e66b844a03a203c3b705ac3764, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T16:16:19,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9e768d9f517a9b5cf0bf527b36f5956b in 482ms, sequenceid=152, compaction requested=true 2024-11-11T16:16:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:16:19,922 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:19,923 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:19,923 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:19,923 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,923 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/42405786a66440ae91aa79d51a6f9b63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/48dc1dbcf4004914bd04a238d411572c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eaa64707151e4ea6b6d809b7d9447571] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=40.3 K 2024-11-11T16:16:19,924 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42405786a66440ae91aa79d51a6f9b63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731341778041 2024-11-11T16:16:19,924 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48dc1dbcf4004914bd04a238d411572c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731341778775 2024-11-11T16:16:19,925 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa64707151e4ea6b6d809b7d9447571, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731341779429 2024-11-11T16:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:19,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T16:16:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:19,931 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:19,932 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/a29e3e9186284acc8592ba9643cffab4 is 50, key is test_row_0/A:col10/1731341779435/Put/seqid=0 2024-11-11T16:16:19,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/fc89903021c149de944da16790ee3a27 is 50, key is test_row_0/A:col10/1731341779565/Put/seqid=0 2024-11-11T16:16:19,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742317_1493 (size=12459) 2024-11-11T16:16:19,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742318_1494 (size=14541) 2024-11-11T16:16:19,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:19,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:19,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:19,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:19,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341839951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341839951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:19,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341839963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,048 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9c8f8fd841764733bbf74f5ccebbd20b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9c8f8fd841764733bbf74f5ccebbd20b 2024-11-11T16:16:20,053 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 9c8f8fd841764733bbf74f5ccebbd20b(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:20,053 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:20,053 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=12, startTime=1731341779172; duration=0sec 2024-11-11T16:16:20,053 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:16:20,053 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:20,053 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:20,053 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:20,054 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:20,054 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:20,054 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,054 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5ae5390c930d43ba810d4a387addbb48, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/37baae1348c441fd9bdadc5b9fd04d29, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c9fdfa88326646c6a945cae45ed9953e] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=35.7 K 2024-11-11T16:16:20,054 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae5390c930d43ba810d4a387addbb48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731341778041 2024-11-11T16:16:20,055 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 37baae1348c441fd9bdadc5b9fd04d29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731341778775 2024-11-11T16:16:20,055 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c9fdfa88326646c6a945cae45ed9953e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731341779435 2024-11-11T16:16:20,071 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:20,071 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/52f2b65cbd3d42929f3126d44433a798 is 50, key is test_row_0/B:col10/1731341779435/Put/seqid=0 2024-11-11T16:16:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341840076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341840076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742319_1495 (size=12459) 2024-11-11T16:16:20,088 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/52f2b65cbd3d42929f3126d44433a798 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/52f2b65cbd3d42929f3126d44433a798 2024-11-11T16:16:20,092 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into 52f2b65cbd3d42929f3126d44433a798(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:20,092 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:20,093 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=13, startTime=1731341779922; duration=0sec 2024-11-11T16:16:20,093 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:20,093 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:20,093 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T16:16:20,095 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T16:16:20,096 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T16:16:20,096 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. because compaction request was cancelled 2024-11-11T16:16:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,096 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:20,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:20,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T16:16:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341840215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341840216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:20,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:20,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341840287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341840287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341840306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,344 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/a29e3e9186284acc8592ba9643cffab4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/a29e3e9186284acc8592ba9643cffab4 2024-11-11T16:16:20,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/fc89903021c149de944da16790ee3a27 2024-11-11T16:16:20,348 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into a29e3e9186284acc8592ba9643cffab4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:20,348 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:20,348 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=13, startTime=1731341779922; duration=0sec 2024-11-11T16:16:20,348 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:20,349 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:20,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/70a5cc9ad46d4a7db375b52ef86e429b is 50, key is test_row_0/B:col10/1731341779565/Put/seqid=0 2024-11-11T16:16:20,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742320_1496 (size=12151) 2024-11-11T16:16:20,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:20,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:20,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:20,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:20,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T16:16:20,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341840626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:20,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:20,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341840735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341840736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:20,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/70a5cc9ad46d4a7db375b52ef86e429b 2024-11-11T16:16:20,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/4e23da5cbc954e9588ef1d1262899fb9 is 50, key is test_row_0/C:col10/1731341779565/Put/seqid=0 2024-11-11T16:16:20,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742321_1497 (size=12151) 2024-11-11T16:16:20,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:20,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:20,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:20,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:21,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:21,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:21,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:21,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:21,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341841126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:21,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341841126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341841146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:21,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/4e23da5cbc954e9588ef1d1262899fb9 2024-11-11T16:16:21,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:21,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:21,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/fc89903021c149de944da16790ee3a27 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/fc89903021c149de944da16790ee3a27 2024-11-11T16:16:21,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/fc89903021c149de944da16790ee3a27, entries=200, sequenceid=191, filesize=14.2 K 2024-11-11T16:16:21,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/70a5cc9ad46d4a7db375b52ef86e429b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/70a5cc9ad46d4a7db375b52ef86e429b 2024-11-11T16:16:21,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/70a5cc9ad46d4a7db375b52ef86e429b, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T16:16:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/4e23da5cbc954e9588ef1d1262899fb9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4e23da5cbc954e9588ef1d1262899fb9 2024-11-11T16:16:21,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4e23da5cbc954e9588ef1d1262899fb9, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T16:16:21,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 9e768d9f517a9b5cf0bf527b36f5956b in 1275ms, sequenceid=191, compaction requested=true 2024-11-11T16:16:21,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:21,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:21,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:21,204 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T16:16:21,204 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T16:16:21,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:21,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:21,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:21,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. because compaction request was cancelled 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. because compaction request was cancelled 2024-11-11T16:16:21,206 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:21,207 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:21,207 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:21,207 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:21,207 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9c8f8fd841764733bbf74f5ccebbd20b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/782002e66b844a03a203c3b705ac3764, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4e23da5cbc954e9588ef1d1262899fb9] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=35.8 K 2024-11-11T16:16:21,208 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c8f8fd841764733bbf74f5ccebbd20b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731341778775 2024-11-11T16:16:21,208 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782002e66b844a03a203c3b705ac3764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731341779435 2024-11-11T16:16:21,208 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e23da5cbc954e9588ef1d1262899fb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731341779563 2024-11-11T16:16:21,215 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:21,215 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/5035462a96d442f5ae1a3af6f8d74c3f is 50, key is test_row_0/C:col10/1731341779565/Put/seqid=0 2024-11-11T16:16:21,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742322_1498 (size=12493) 2024-11-11T16:16:21,250 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/5035462a96d442f5ae1a3af6f8d74c3f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/5035462a96d442f5ae1a3af6f8d74c3f 2024-11-11T16:16:21,258 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 5035462a96d442f5ae1a3af6f8d74c3f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:21,258 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:21,259 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=13, startTime=1731341781204; duration=0sec 2024-11-11T16:16:21,259 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:21,259 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:21,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T16:16:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:21,326 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T16:16:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/972c68f87bd94962a7cfc1891d5732ab is 50, key is test_row_0/A:col10/1731341779961/Put/seqid=0 2024-11-11T16:16:21,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742323_1499 (size=9757) 2024-11-11T16:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T16:16:21,746 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/972c68f87bd94962a7cfc1891d5732ab 2024-11-11T16:16:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/5dc6ff9239c14135a4f1efb47afa1261 is 50, key is test_row_0/B:col10/1731341779961/Put/seqid=0 2024-11-11T16:16:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:21,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:21,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742324_1500 (size=9757) 2024-11-11T16:16:21,760 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/5dc6ff9239c14135a4f1efb47afa1261 2024-11-11T16:16:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/a3269f987cff436db86e7adde1af04b1 is 50, key is test_row_0/C:col10/1731341779961/Put/seqid=0 2024-11-11T16:16:21,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742325_1501 (size=9757) 2024-11-11T16:16:21,774 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/a3269f987cff436db86e7adde1af04b1 2024-11-11T16:16:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/972c68f87bd94962a7cfc1891d5732ab as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/972c68f87bd94962a7cfc1891d5732ab 2024-11-11T16:16:21,781 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/972c68f87bd94962a7cfc1891d5732ab, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T16:16:21,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/5dc6ff9239c14135a4f1efb47afa1261 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5dc6ff9239c14135a4f1efb47afa1261 2024-11-11T16:16:21,784 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5dc6ff9239c14135a4f1efb47afa1261, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T16:16:21,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/a3269f987cff436db86e7adde1af04b1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a3269f987cff436db86e7adde1af04b1 2024-11-11T16:16:21,789 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a3269f987cff436db86e7adde1af04b1, entries=100, sequenceid=204, filesize=9.5 K 2024-11-11T16:16:21,789 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=53.67 KB/54960 for 9e768d9f517a9b5cf0bf527b36f5956b in 463ms, sequenceid=204, compaction requested=true 2024-11-11T16:16:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T16:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T16:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:21,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T16:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:21,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:21,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T16:16:21,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3080 sec 2024-11-11T16:16:21,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b9b161b88b2846ea8554fbb7705030c1 is 50, key is test_row_0/A:col10/1731341781790/Put/seqid=0 2024-11-11T16:16:21,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.3170 sec 2024-11-11T16:16:21,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742326_1502 (size=14541) 2024-11-11T16:16:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341841879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:21,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341841882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:21,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:21,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341841995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341842006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341842136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,142 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341842146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,156 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341842165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,171 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b9b161b88b2846ea8554fbb7705030c1 2024-11-11T16:16:22,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e2fe792e67b409d8ae4dd60b9efcfe9 is 50, key is test_row_0/B:col10/1731341781790/Put/seqid=0 2024-11-11T16:16:22,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742327_1503 (size=12151) 2024-11-11T16:16:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341842216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341842216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341842536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341842536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:22,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e2fe792e67b409d8ae4dd60b9efcfe9 2024-11-11T16:16:22,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/f741fe13a71c4b0b8c693da3ef5ff052 is 50, key is test_row_0/C:col10/1731341781790/Put/seqid=0 2024-11-11T16:16:22,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742328_1504 (size=12151) 2024-11-11T16:16:23,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/f741fe13a71c4b0b8c693da3ef5ff052 2024-11-11T16:16:23,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b9b161b88b2846ea8554fbb7705030c1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b9b161b88b2846ea8554fbb7705030c1 2024-11-11T16:16:23,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b9b161b88b2846ea8554fbb7705030c1, entries=200, sequenceid=216, filesize=14.2 K 2024-11-11T16:16:23,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e2fe792e67b409d8ae4dd60b9efcfe9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2fe792e67b409d8ae4dd60b9efcfe9 2024-11-11T16:16:23,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2fe792e67b409d8ae4dd60b9efcfe9, entries=150, sequenceid=216, filesize=11.9 K 2024-11-11T16:16:23,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/f741fe13a71c4b0b8c693da3ef5ff052 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/f741fe13a71c4b0b8c693da3ef5ff052 2024-11-11T16:16:23,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/f741fe13a71c4b0b8c693da3ef5ff052, entries=150, sequenceid=216, filesize=11.9 K 2024-11-11T16:16:23,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9e768d9f517a9b5cf0bf527b36f5956b in 1252ms, sequenceid=216, compaction requested=true 2024-11-11T16:16:23,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:23,045 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:23,045 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:23,046 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:23,046 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:23,046 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:23,046 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:23,046 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:23,046 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:23,046 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/52f2b65cbd3d42929f3126d44433a798, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/70a5cc9ad46d4a7db375b52ef86e429b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5dc6ff9239c14135a4f1efb47afa1261, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2fe792e67b409d8ae4dd60b9efcfe9] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=45.4 K 2024-11-11T16:16:23,046 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/a29e3e9186284acc8592ba9643cffab4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/fc89903021c149de944da16790ee3a27, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/972c68f87bd94962a7cfc1891d5732ab, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b9b161b88b2846ea8554fbb7705030c1] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=50.1 K 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 52f2b65cbd3d42929f3126d44433a798, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731341779435 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a29e3e9186284acc8592ba9643cffab4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731341779435 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc89903021c149de944da16790ee3a27, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731341779563 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a5cc9ad46d4a7db375b52ef86e429b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731341779563 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 972c68f87bd94962a7cfc1891d5732ab, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731341779949 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc6ff9239c14135a4f1efb47afa1261, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731341779949 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9b161b88b2846ea8554fbb7705030c1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731341781763 2024-11-11T16:16:23,047 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2fe792e67b409d8ae4dd60b9efcfe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731341781763 2024-11-11T16:16:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:23,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341781848, max timestamp=1731341783052Num uniques 859; 2024-11-11T16:16:23,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:23,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341781848, max timestamp=1731341783052Num uniques 859; 2024-11-11T16:16:23,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341781848, max timestamp=1731341783052Num uniques 859; 2024-11-11T16:16:23,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:23,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:23,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-11T16:16:23,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:23,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:23,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:23,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:23,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:23,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:23,058 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:23,058 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:23,059 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/acd47fb048ce44e89bf4434503708cc6 is 50, key is test_row_0/A:col10/1731341781790/Put/seqid=0 2024-11-11T16:16:23,059 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/976cefec078c49e0a69e02c35cc1d21e is 50, key is test_row_0/B:col10/1731341781790/Put/seqid=0 2024-11-11T16:16:23,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742329_1505 (size=12595) 2024-11-11T16:16:23,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/18357dbd70b5436397bbaf1419bba73c is 50, key is test_row_0/A:col10/1731341783054/Put/seqid=0 2024-11-11T16:16:23,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742331_1507 (size=16931) 2024-11-11T16:16:23,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742330_1506 (size=12595) 2024-11-11T16:16:23,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341783102, max timestamp=1731341783149Num uniques 859; 2024-11-11T16:16:23,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:23,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341783102, max timestamp=1731341783149Num uniques 859; 2024-11-11T16:16:23,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341783102, max timestamp=1731341783149Num uniques 859; 2024-11-11T16:16:23,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:23,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:23,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731341783231, max timestamp=1731341783280Num uniques 1068; 2024-11-11T16:16:23,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T16:16:23,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731341783231, max timestamp=1731341783280Num uniques 1068; 2024-11-11T16:16:23,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T16:16:23,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731341783231, max timestamp=1731341783280Num uniques 1068; 2024-11-11T16:16:23,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T16:16:23,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341783377, max timestamp=1731341783439Num uniques 1077; 2024-11-11T16:16:23,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341783377, max timestamp=1731341783439Num uniques 1077; 2024-11-11T16:16:23,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T16:16:23,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T16:16:23,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341783377, max timestamp=1731341783439Num uniques 1077; 2024-11-11T16:16:23,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T16:16:23,468 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/976cefec078c49e0a69e02c35cc1d21e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/976cefec078c49e0a69e02c35cc1d21e 2024-11-11T16:16:23,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/18357dbd70b5436397bbaf1419bba73c 2024-11-11T16:16:23,473 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into 976cefec078c49e0a69e02c35cc1d21e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:23,473 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:23,473 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=12, startTime=1731341783045; duration=0sec 2024-11-11T16:16:23,473 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:23,473 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:23,473 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:23,479 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:23,479 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:23,479 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:23,479 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/5035462a96d442f5ae1a3af6f8d74c3f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a3269f987cff436db86e7adde1af04b1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/f741fe13a71c4b0b8c693da3ef5ff052] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=33.6 K 2024-11-11T16:16:23,479 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5035462a96d442f5ae1a3af6f8d74c3f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731341779563 2024-11-11T16:16:23,480 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a3269f987cff436db86e7adde1af04b1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731341779949 2024-11-11T16:16:23,480 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f741fe13a71c4b0b8c693da3ef5ff052, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731341781763 2024-11-11T16:16:23,482 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/acd47fb048ce44e89bf4434503708cc6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/acd47fb048ce44e89bf4434503708cc6 2024-11-11T16:16:23,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/b4285c416ecb414bb32aa03cbc2e854d is 50, key is test_row_0/B:col10/1731341783054/Put/seqid=0 2024-11-11T16:16:23,489 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into acd47fb048ce44e89bf4434503708cc6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:23,489 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:23,489 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=12, startTime=1731341783045; duration=0sec 2024-11-11T16:16:23,489 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:23,489 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742332_1508 (size=12151) 2024-11-11T16:16:23,492 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:23,492 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/145112af48924b319ee6c2af44beb310 is 50, key is test_row_0/C:col10/1731341781790/Put/seqid=0 2024-11-11T16:16:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742333_1509 (size=12595) 2024-11-11T16:16:23,501 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/145112af48924b319ee6c2af44beb310 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/145112af48924b319ee6c2af44beb310 2024-11-11T16:16:23,505 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 145112af48924b319ee6c2af44beb310(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:23,505 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:23,505 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=13, startTime=1731341783045; duration=0sec 2024-11-11T16:16:23,505 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:23,505 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731341783495, max timestamp=1731341783550Num uniques 1086; 2024-11-11T16:16:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731341783495, max timestamp=1731341783550Num uniques 1086; 2024-11-11T16:16:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T16:16:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T16:16:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731341783495, max timestamp=1731341783550Num uniques 1086; 2024-11-11T16:16:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T16:16:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T16:16:23,636 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:23,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T16:16:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T16:16:23,638 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:23,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:23,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:23,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341783624, max timestamp=1731341783673Num uniques 1095; 2024-11-11T16:16:23,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-11T16:16:23,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341783624, max timestamp=1731341783673Num uniques 1095; 2024-11-11T16:16:23,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-11T16:16:23,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731341783624, max timestamp=1731341783673Num uniques 1095; 2024-11-11T16:16:23,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-11T16:16:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T16:16:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341843770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:23,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:23,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731341783751, max timestamp=1731341783776Num uniques 1104; 2024-11-11T16:16:23,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731341783751, max timestamp=1731341783776Num uniques 1104; 2024-11-11T16:16:23,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-11T16:16:23,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-11T16:16:23,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731341783751, max timestamp=1731341783776Num uniques 1104; 2024-11-11T16:16:23,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-11T16:16:23,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:23,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:23,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/b4285c416ecb414bb32aa03cbc2e854d 2024-11-11T16:16:23,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9d908c8a2bb746599a47106b5dcdd6fe is 50, key is test_row_0/C:col10/1731341783054/Put/seqid=0 2024-11-11T16:16:23,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742334_1510 (size=12151) 2024-11-11T16:16:23,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9d908c8a2bb746599a47106b5dcdd6fe 2024-11-11T16:16:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/18357dbd70b5436397bbaf1419bba73c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/18357dbd70b5436397bbaf1419bba73c 2024-11-11T16:16:23,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/18357dbd70b5436397bbaf1419bba73c, entries=250, sequenceid=238, filesize=16.5 K 2024-11-11T16:16:23,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/b4285c416ecb414bb32aa03cbc2e854d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b4285c416ecb414bb32aa03cbc2e854d 2024-11-11T16:16:23,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b4285c416ecb414bb32aa03cbc2e854d, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T16:16:23,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9d908c8a2bb746599a47106b5dcdd6fe as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d908c8a2bb746599a47106b5dcdd6fe 2024-11-11T16:16:23,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d908c8a2bb746599a47106b5dcdd6fe, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T16:16:23,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=140.89 KB/144270 for 9e768d9f517a9b5cf0bf527b36f5956b in 862ms, sequenceid=238, compaction requested=false 2024-11-11T16:16:23,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:23,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=147.60 KB heapSize=421.31 KB 2024-11-11T16:16:23,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731341783898, max timestamp=1731341783920Num uniques 1063; 2024-11-11T16:16:23,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-11T16:16:23,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731341783898, max timestamp=1731341783920Num uniques 1063; 2024-11-11T16:16:23,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-11T16:16:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T16:16:23,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/62b08659ad2442dc9c79e5a2e4c2e2d6 is 50, key is test_row_0/A:col10/1731341783921/Put/seqid=0 2024-11-11T16:16:23,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742335_1511 (size=17181) 2024-11-11T16:16:23,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/62b08659ad2442dc9c79e5a2e4c2e2d6 2024-11-11T16:16:23,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:23,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:23,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/92cb0da5ca724faa8b87b64bd836533f is 50, key is test_row_0/B:col10/1731341783921/Put/seqid=0 2024-11-11T16:16:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T16:16:23,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742336_1512 (size=12301) 2024-11-11T16:16:24,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341844031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341844033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341844146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341844146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341844146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,154 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341844186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341844186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,195 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,195 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:24,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:24,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T16:16:24,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/92cb0da5ca724faa8b87b64bd836533f 2024-11-11T16:16:24,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/79b1f6dcd1404582a39723f98c0dd6bc is 50, key is test_row_0/C:col10/1731341783921/Put/seqid=0 2024-11-11T16:16:24,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341844366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341844366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742337_1513 (size=12301) 2024-11-11T16:16:24,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:24,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341844675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341844676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:24,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:24,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/79b1f6dcd1404582a39723f98c0dd6bc 2024-11-11T16:16:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T16:16:24,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/62b08659ad2442dc9c79e5a2e4c2e2d6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/62b08659ad2442dc9c79e5a2e4c2e2d6 2024-11-11T16:16:24,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/62b08659ad2442dc9c79e5a2e4c2e2d6, entries=250, sequenceid=366, filesize=16.8 K 2024-11-11T16:16:24,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/92cb0da5ca724faa8b87b64bd836533f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/92cb0da5ca724faa8b87b64bd836533f 2024-11-11T16:16:24,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/92cb0da5ca724faa8b87b64bd836533f, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T16:16:24,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/79b1f6dcd1404582a39723f98c0dd6bc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/79b1f6dcd1404582a39723f98c0dd6bc 2024-11-11T16:16:24,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/79b1f6dcd1404582a39723f98c0dd6bc, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T16:16:24,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~204.22 KB/209120, currentSize=107.34 KB/109920 for 9e768d9f517a9b5cf0bf527b36f5956b in 866ms, sequenceid=366, compaction requested=true 2024-11-11T16:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:24,789 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:24,789 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:24,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:24,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:24,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:24,790 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 46707 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:24,790 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:24,790 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:24,790 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:24,790 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,790 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,790 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/976cefec078c49e0a69e02c35cc1d21e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b4285c416ecb414bb32aa03cbc2e854d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/92cb0da5ca724faa8b87b64bd836533f] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=36.2 K 2024-11-11T16:16:24,790 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/acd47fb048ce44e89bf4434503708cc6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/18357dbd70b5436397bbaf1419bba73c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/62b08659ad2442dc9c79e5a2e4c2e2d6] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=45.6 K 2024-11-11T16:16:24,791 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 976cefec078c49e0a69e02c35cc1d21e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731341781763 2024-11-11T16:16:24,791 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd47fb048ce44e89bf4434503708cc6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731341781763 2024-11-11T16:16:24,791 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b4285c416ecb414bb32aa03cbc2e854d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731341781848 2024-11-11T16:16:24,791 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18357dbd70b5436397bbaf1419bba73c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731341781848 2024-11-11T16:16:24,792 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b08659ad2442dc9c79e5a2e4c2e2d6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341783898 2024-11-11T16:16:24,792 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 92cb0da5ca724faa8b87b64bd836533f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341783902 2024-11-11T16:16:24,799 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:24,799 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/5f83a823d610490c91ca9f51ce15958e is 50, key is test_row_0/B:col10/1731341783921/Put/seqid=0 2024-11-11T16:16:24,799 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:24,800 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/82b572c2329e4bfbba7a5e869b098ade is 50, key is test_row_0/A:col10/1731341783921/Put/seqid=0 2024-11-11T16:16:24,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742339_1515 (size=12847) 2024-11-11T16:16:24,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742338_1514 (size=12847) 2024-11-11T16:16:24,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T16:16:24,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:24,863 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:16:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/05ada2d6b79e4618a4cf58705c944890 is 50, key is test_row_0/A:col10/1731341784022/Put/seqid=0 2024-11-11T16:16:24,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742340_1516 (size=12301) 2024-11-11T16:16:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:25,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:25,212 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/82b572c2329e4bfbba7a5e869b098ade as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/82b572c2329e4bfbba7a5e869b098ade 2024-11-11T16:16:25,216 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/5f83a823d610490c91ca9f51ce15958e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5f83a823d610490c91ca9f51ce15958e 2024-11-11T16:16:25,218 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into 82b572c2329e4bfbba7a5e869b098ade(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:25,218 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:25,218 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=13, startTime=1731341784789; duration=0sec 2024-11-11T16:16:25,218 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:25,218 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:25,219 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:25,220 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into 5f83a823d610490c91ca9f51ce15958e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:25,220 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:25,220 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=13, startTime=1731341784789; duration=0sec 2024-11-11T16:16:25,220 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:25,220 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:25,221 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:25,221 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:25,221 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:25,221 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/145112af48924b319ee6c2af44beb310, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d908c8a2bb746599a47106b5dcdd6fe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/79b1f6dcd1404582a39723f98c0dd6bc] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=36.2 K 2024-11-11T16:16:25,221 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145112af48924b319ee6c2af44beb310, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731341781763 2024-11-11T16:16:25,222 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d908c8a2bb746599a47106b5dcdd6fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731341781848 2024-11-11T16:16:25,222 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b1f6dcd1404582a39723f98c0dd6bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341783902 2024-11-11T16:16:25,227 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:25,227 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/013a170920c24917b7261aae9da29920 is 50, key is test_row_0/C:col10/1731341783921/Put/seqid=0 2024-11-11T16:16:25,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742341_1517 (size=12847) 2024-11-11T16:16:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341845243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:25,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341845253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:25,292 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/05ada2d6b79e4618a4cf58705c944890 2024-11-11T16:16:25,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/979bd7bba44749b984dc9c12331abc29 is 50, key is test_row_0/B:col10/1731341784022/Put/seqid=0 2024-11-11T16:16:25,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742342_1518 (size=12301) 2024-11-11T16:16:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341845356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:25,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341845375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341845576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:25,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341845585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:25,651 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/013a170920c24917b7261aae9da29920 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/013a170920c24917b7261aae9da29920 2024-11-11T16:16:25,659 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 013a170920c24917b7261aae9da29920(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:25,659 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:25,659 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=13, startTime=1731341784790; duration=0sec 2024-11-11T16:16:25,659 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:25,660 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:25,706 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/979bd7bba44749b984dc9c12331abc29 2024-11-11T16:16:25,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/4af36e1f356e468992aea45e71c79fb5 is 50, key is test_row_0/C:col10/1731341784022/Put/seqid=0 2024-11-11T16:16:25,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742343_1519 (size=12301) 2024-11-11T16:16:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T16:16:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341845886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341845906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:26,117 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/4af36e1f356e468992aea45e71c79fb5 2024-11-11T16:16:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/05ada2d6b79e4618a4cf58705c944890 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/05ada2d6b79e4618a4cf58705c944890 2024-11-11T16:16:26,124 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/05ada2d6b79e4618a4cf58705c944890, entries=150, sequenceid=385, filesize=12.0 K 2024-11-11T16:16:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/979bd7bba44749b984dc9c12331abc29 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/979bd7bba44749b984dc9c12331abc29 2024-11-11T16:16:26,128 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/979bd7bba44749b984dc9c12331abc29, entries=150, sequenceid=385, filesize=12.0 K 2024-11-11T16:16:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/4af36e1f356e468992aea45e71c79fb5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4af36e1f356e468992aea45e71c79fb5 2024-11-11T16:16:26,131 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4af36e1f356e468992aea45e71c79fb5, entries=150, sequenceid=385, filesize=12.0 K 2024-11-11T16:16:26,132 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9e768d9f517a9b5cf0bf527b36f5956b in 1269ms, sequenceid=385, compaction requested=false 2024-11-11T16:16:26,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:26,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:26,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T16:16:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T16:16:26,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T16:16:26,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4940 sec 2024-11-11T16:16:26,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.4990 sec 2024-11-11T16:16:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:26,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T16:16:26,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:26,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:26,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:26,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:26,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:26,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:26,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/e3936543ab7e4fc28aeed876779c66a5 is 50, key is test_row_0/A:col10/1731341786410/Put/seqid=0 2024-11-11T16:16:26,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742344_1520 (size=14741) 2024-11-11T16:16:26,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341846504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:26,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341846505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341846615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341846626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:26,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/e3936543ab7e4fc28aeed876779c66a5 2024-11-11T16:16:26,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/e256bd9bbc03438b8c61dc2185ae6c62 is 50, key is test_row_0/B:col10/1731341786410/Put/seqid=0 2024-11-11T16:16:26,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341846837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:26,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341846857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:26,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742345_1521 (size=12301) 2024-11-11T16:16:26,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/e256bd9bbc03438b8c61dc2185ae6c62 2024-11-11T16:16:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:26,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:26,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/633d2a4c87454db08f5eb3a9aece8d5b is 50, key is test_row_0/C:col10/1731341786410/Put/seqid=0 2024-11-11T16:16:26,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742346_1522 (size=12301) 2024-11-11T16:16:27,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341847156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:27,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341847176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:27,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/633d2a4c87454db08f5eb3a9aece8d5b 2024-11-11T16:16:27,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/e3936543ab7e4fc28aeed876779c66a5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/e3936543ab7e4fc28aeed876779c66a5 2024-11-11T16:16:27,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/e3936543ab7e4fc28aeed876779c66a5, entries=200, sequenceid=402, filesize=14.4 K 2024-11-11T16:16:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/e256bd9bbc03438b8c61dc2185ae6c62 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/e256bd9bbc03438b8c61dc2185ae6c62 2024-11-11T16:16:27,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/e256bd9bbc03438b8c61dc2185ae6c62, entries=150, sequenceid=402, filesize=12.0 K 2024-11-11T16:16:27,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/633d2a4c87454db08f5eb3a9aece8d5b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/633d2a4c87454db08f5eb3a9aece8d5b 2024-11-11T16:16:27,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/633d2a4c87454db08f5eb3a9aece8d5b, entries=150, sequenceid=402, filesize=12.0 K 2024-11-11T16:16:27,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9e768d9f517a9b5cf0bf527b36f5956b in 940ms, sequenceid=402, compaction requested=true 2024-11-11T16:16:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:27,351 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:27,351 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:27,352 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:27,352 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:27,352 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:27,352 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:27,352 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:27,352 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:27,352 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5f83a823d610490c91ca9f51ce15958e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/979bd7bba44749b984dc9c12331abc29, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/e256bd9bbc03438b8c61dc2185ae6c62] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=36.6 K 2024-11-11T16:16:27,352 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/82b572c2329e4bfbba7a5e869b098ade, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/05ada2d6b79e4618a4cf58705c944890, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/e3936543ab7e4fc28aeed876779c66a5] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=39.0 K 2024-11-11T16:16:27,353 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82b572c2329e4bfbba7a5e869b098ade, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341783902 2024-11-11T16:16:27,353 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f83a823d610490c91ca9f51ce15958e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341783902 2024-11-11T16:16:27,353 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ada2d6b79e4618a4cf58705c944890, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731341784022 2024-11-11T16:16:27,353 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 979bd7bba44749b984dc9c12331abc29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731341784022 2024-11-11T16:16:27,354 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3936543ab7e4fc28aeed876779c66a5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731341785232 2024-11-11T16:16:27,354 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e256bd9bbc03438b8c61dc2185ae6c62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731341785232 2024-11-11T16:16:27,360 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:27,361 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/31c873bee8a04e4ab2822c9cb03de47f is 50, key is test_row_0/A:col10/1731341786410/Put/seqid=0 2024-11-11T16:16:27,367 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:27,367 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ae1d8a9f24b54833aaf2f3da4c732d80 is 50, key is test_row_0/B:col10/1731341786410/Put/seqid=0 2024-11-11T16:16:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742347_1523 (size=12949) 2024-11-11T16:16:27,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742348_1524 (size=12949) 2024-11-11T16:16:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:27,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:16:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:27,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:27,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b391760faeb74e0294a678297929ef84 is 50, key is test_row_0/A:col10/1731341786503/Put/seqid=0 2024-11-11T16:16:27,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742349_1525 (size=14741) 2024-11-11T16:16:27,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341847745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:27,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341847746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:27,772 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/31c873bee8a04e4ab2822c9cb03de47f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/31c873bee8a04e4ab2822c9cb03de47f 2024-11-11T16:16:27,776 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into 31c873bee8a04e4ab2822c9cb03de47f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:27,776 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:27,776 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=13, startTime=1731341787351; duration=0sec 2024-11-11T16:16:27,777 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:27,777 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:27,777 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:27,778 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:27,778 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:27,779 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:27,779 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/013a170920c24917b7261aae9da29920, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4af36e1f356e468992aea45e71c79fb5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/633d2a4c87454db08f5eb3a9aece8d5b] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=36.6 K 2024-11-11T16:16:27,779 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013a170920c24917b7261aae9da29920, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731341783902 2024-11-11T16:16:27,780 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af36e1f356e468992aea45e71c79fb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731341784022 2024-11-11T16:16:27,780 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 633d2a4c87454db08f5eb3a9aece8d5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731341785232 2024-11-11T16:16:27,783 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ae1d8a9f24b54833aaf2f3da4c732d80 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ae1d8a9f24b54833aaf2f3da4c732d80 2024-11-11T16:16:27,789 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into ae1d8a9f24b54833aaf2f3da4c732d80(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:27,789 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:27,789 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=13, startTime=1731341787351; duration=0sec 2024-11-11T16:16:27,789 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:27,789 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:27,789 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:27,789 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/356a9fa87b9d426a894b33096d6b923d is 50, key is test_row_0/C:col10/1731341786410/Put/seqid=0 2024-11-11T16:16:27,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742350_1526 (size=12949) 2024-11-11T16:16:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T16:16:27,797 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:27,799 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T16:16:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T16:16:27,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:27,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:27,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:27,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341847866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:27,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341847866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T16:16:27,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T16:16:27,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:27,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:27,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:27,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:27,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b391760faeb74e0294a678297929ef84 2024-11-11T16:16:28,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341848089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:28,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341848089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:28,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/4ae19636bebe4348a536f799ab515212 is 50, key is test_row_0/B:col10/1731341786503/Put/seqid=0 2024-11-11T16:16:28,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742351_1527 (size=12301) 2024-11-11T16:16:28,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/4ae19636bebe4348a536f799ab515212 2024-11-11T16:16:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T16:16:28,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/a74d4e5f80ca489b909046e9f45833e3 is 50, key is test_row_0/C:col10/1731341786503/Put/seqid=0 2024-11-11T16:16:28,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T16:16:28,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:28,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:28,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:28,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742352_1528 (size=12301) 2024-11-11T16:16:28,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33476 deadline: 1731341848166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:28,181 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:28,204 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/356a9fa87b9d426a894b33096d6b923d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/356a9fa87b9d426a894b33096d6b923d 2024-11-11T16:16:28,210 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 356a9fa87b9d426a894b33096d6b923d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:28,210 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:28,210 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=13, startTime=1731341787351; duration=0sec 2024-11-11T16:16:28,210 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:28,210 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:28,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1731341848216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:28,223 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:28,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:28,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731341848226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:28,236 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_2' in 9e768d9f517a9b5cf0bf527b36f5956b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:28,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T16:16:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:28,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341848405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341848405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T16:16:28,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T16:16:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:28,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:28,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/a74d4e5f80ca489b909046e9f45833e3 2024-11-11T16:16:28,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/b391760faeb74e0294a678297929ef84 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b391760faeb74e0294a678297929ef84 2024-11-11T16:16:28,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b391760faeb74e0294a678297929ef84, entries=200, sequenceid=421, filesize=14.4 K 2024-11-11T16:16:28,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/4ae19636bebe4348a536f799ab515212 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/4ae19636bebe4348a536f799ab515212 2024-11-11T16:16:28,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/4ae19636bebe4348a536f799ab515212, entries=150, sequenceid=421, filesize=12.0 K 2024-11-11T16:16:28,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/a74d4e5f80ca489b909046e9f45833e3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a74d4e5f80ca489b909046e9f45833e3 2024-11-11T16:16:28,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a74d4e5f80ca489b909046e9f45833e3, entries=150, sequenceid=421, filesize=12.0 K 2024-11-11T16:16:28,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9e768d9f517a9b5cf0bf527b36f5956b in 875ms, sequenceid=421, compaction requested=false 2024-11-11T16:16:28,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:28,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T16:16:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:28,593 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:16:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:28,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/ab912b4b6c03454d9ef5301efbc2c707 is 50, key is test_row_0/A:col10/1731341787743/Put/seqid=0 2024-11-11T16:16:28,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742353_1529 (size=12301) 2024-11-11T16:16:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T16:16:29,008 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/ab912b4b6c03454d9ef5301efbc2c707 2024-11-11T16:16:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/180c123a0bf940dea4b3060d60ad6ecb is 50, key is test_row_0/B:col10/1731341787743/Put/seqid=0 2024-11-11T16:16:29,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742354_1530 (size=12301) 2024-11-11T16:16:29,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341849046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341849062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341849166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341849186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341849386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341849405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,433 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/180c123a0bf940dea4b3060d60ad6ecb 2024-11-11T16:16:29,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/3ece60c6f2a543caba2a42b673009e46 is 50, key is test_row_0/C:col10/1731341787743/Put/seqid=0 2024-11-11T16:16:29,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742355_1531 (size=12301) 2024-11-11T16:16:29,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341849706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341849725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:29,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:29,845 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/3ece60c6f2a543caba2a42b673009e46 2024-11-11T16:16:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/ab912b4b6c03454d9ef5301efbc2c707 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/ab912b4b6c03454d9ef5301efbc2c707 2024-11-11T16:16:29,853 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/ab912b4b6c03454d9ef5301efbc2c707, entries=150, sequenceid=437, filesize=12.0 K 2024-11-11T16:16:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/180c123a0bf940dea4b3060d60ad6ecb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/180c123a0bf940dea4b3060d60ad6ecb 2024-11-11T16:16:29,857 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/180c123a0bf940dea4b3060d60ad6ecb, entries=150, sequenceid=437, filesize=12.0 K 2024-11-11T16:16:29,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/3ece60c6f2a543caba2a42b673009e46 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/3ece60c6f2a543caba2a42b673009e46 2024-11-11T16:16:29,861 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/3ece60c6f2a543caba2a42b673009e46, entries=150, sequenceid=437, filesize=12.0 K 2024-11-11T16:16:29,862 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9e768d9f517a9b5cf0bf527b36f5956b in 1269ms, sequenceid=437, compaction requested=true 2024-11-11T16:16:29,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:29,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:29,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T16:16:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T16:16:29,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T16:16:29,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0500 sec 2024-11-11T16:16:29,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.0660 sec 2024-11-11T16:16:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T16:16:29,956 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:29,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T16:16:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T16:16:29,959 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:29,959 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:29,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T16:16:30,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T16:16:30,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:30,114 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:16:30,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:30,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:30,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:30,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:30,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:30,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:30,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/d1d5823fd14c455397ffabf344b45ef2 is 50, key is test_row_0/A:col10/1731341789059/Put/seqid=0 2024-11-11T16:16:30,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742356_1532 (size=12301) 2024-11-11T16:16:30,122 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/d1d5823fd14c455397ffabf344b45ef2 2024-11-11T16:16:30,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/62ae2f10ab4f4beeb4e78e35cb8a86b1 is 50, key is test_row_0/B:col10/1731341789059/Put/seqid=0 2024-11-11T16:16:30,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742357_1533 (size=12301) 2024-11-11T16:16:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:30,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T16:16:30,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341850271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341850279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341850386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341850395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,538 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/62ae2f10ab4f4beeb4e78e35cb8a86b1 2024-11-11T16:16:30,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/2e90c0ae84804a1b8bedcf2b9973a33f is 50, key is test_row_0/C:col10/1731341789059/Put/seqid=0 2024-11-11T16:16:30,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742358_1534 (size=12301) 2024-11-11T16:16:30,552 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/2e90c0ae84804a1b8bedcf2b9973a33f 2024-11-11T16:16:30,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/d1d5823fd14c455397ffabf344b45ef2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d1d5823fd14c455397ffabf344b45ef2 2024-11-11T16:16:30,558 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d1d5823fd14c455397ffabf344b45ef2, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T16:16:30,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/62ae2f10ab4f4beeb4e78e35cb8a86b1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/62ae2f10ab4f4beeb4e78e35cb8a86b1 2024-11-11T16:16:30,562 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/62ae2f10ab4f4beeb4e78e35cb8a86b1, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T16:16:30,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/2e90c0ae84804a1b8bedcf2b9973a33f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2e90c0ae84804a1b8bedcf2b9973a33f 2024-11-11T16:16:30,568 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2e90c0ae84804a1b8bedcf2b9973a33f, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T16:16:30,569 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9e768d9f517a9b5cf0bf527b36f5956b in 455ms, sequenceid=456, compaction requested=true 2024-11-11T16:16:30,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:30,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:30,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T16:16:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T16:16:30,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T16:16:30,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 611 msec 2024-11-11T16:16:30,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 615 msec 2024-11-11T16:16:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T16:16:30,586 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:30,587 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T16:16:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T16:16:30,588 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:30,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:30,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:30,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T16:16:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:30,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/dd03f19440d04dfc8e3f0ee8940e397d is 50, key is test_row_0/A:col10/1731341790598/Put/seqid=0 2024-11-11T16:16:30,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742359_1535 (size=14741) 2024-11-11T16:16:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T16:16:30,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341850693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341850696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:30,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:30,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341850816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341850816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:30,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:30,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:30,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T16:16:31,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/dd03f19440d04dfc8e3f0ee8940e397d 2024-11-11T16:16:31,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/7ef200c2d52742bd86c6079467aa09de is 50, key is test_row_0/B:col10/1731341790598/Put/seqid=0 2024-11-11T16:16:31,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742360_1536 (size=12301) 2024-11-11T16:16:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341851036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341851036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:31,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T16:16:31,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341851356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:31,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341851356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:31,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/7ef200c2d52742bd86c6079467aa09de 2024-11-11T16:16:31,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/242967ee1cf545118326f25e2d002747 is 50, key is test_row_0/C:col10/1731341790598/Put/seqid=0 2024-11-11T16:16:31,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742361_1537 (size=12301) 2024-11-11T16:16:31,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:31,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:31,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:31,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T16:16:31,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/242967ee1cf545118326f25e2d002747 2024-11-11T16:16:31,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/dd03f19440d04dfc8e3f0ee8940e397d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/dd03f19440d04dfc8e3f0ee8940e397d 2024-11-11T16:16:31,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/dd03f19440d04dfc8e3f0ee8940e397d, entries=200, sequenceid=470, filesize=14.4 K 2024-11-11T16:16:31,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/7ef200c2d52742bd86c6079467aa09de as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/7ef200c2d52742bd86c6079467aa09de 2024-11-11T16:16:31,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/7ef200c2d52742bd86c6079467aa09de, entries=150, sequenceid=470, filesize=12.0 K 2024-11-11T16:16:31,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/242967ee1cf545118326f25e2d002747 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/242967ee1cf545118326f25e2d002747 2024-11-11T16:16:31,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/242967ee1cf545118326f25e2d002747, entries=150, sequenceid=470, filesize=12.0 K 2024-11-11T16:16:31,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9e768d9f517a9b5cf0bf527b36f5956b in 1246ms, sequenceid=470, compaction requested=true 2024-11-11T16:16:31,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:31,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:31,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:31,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:31,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:31,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:31,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:31,846 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T16:16:31,846 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T16:16:31,847 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T16:16:31,847 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67033 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T16:16:31,847 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:31,847 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:31,847 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,847 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,847 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ae1d8a9f24b54833aaf2f3da4c732d80, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/4ae19636bebe4348a536f799ab515212, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/180c123a0bf940dea4b3060d60ad6ecb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/62ae2f10ab4f4beeb4e78e35cb8a86b1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/7ef200c2d52742bd86c6079467aa09de] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=60.7 K 2024-11-11T16:16:31,847 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/31c873bee8a04e4ab2822c9cb03de47f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b391760faeb74e0294a678297929ef84, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/ab912b4b6c03454d9ef5301efbc2c707, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d1d5823fd14c455397ffabf344b45ef2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/dd03f19440d04dfc8e3f0ee8940e397d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=65.5 K 2024-11-11T16:16:31,848 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1d8a9f24b54833aaf2f3da4c732d80, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731341785232 2024-11-11T16:16:31,848 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae19636bebe4348a536f799ab515212, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731341786478 2024-11-11T16:16:31,848 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31c873bee8a04e4ab2822c9cb03de47f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731341785232 2024-11-11T16:16:31,848 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting b391760faeb74e0294a678297929ef84, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731341786478 2024-11-11T16:16:31,848 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 180c123a0bf940dea4b3060d60ad6ecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731341787727 2024-11-11T16:16:31,848 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab912b4b6c03454d9ef5301efbc2c707, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731341787727 2024-11-11T16:16:31,849 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ae2f10ab4f4beeb4e78e35cb8a86b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731341788976 2024-11-11T16:16:31,849 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1d5823fd14c455397ffabf344b45ef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731341788976 2024-11-11T16:16:31,849 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef200c2d52742bd86c6079467aa09de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731341790270 2024-11-11T16:16:31,849 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd03f19440d04dfc8e3f0ee8940e397d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731341790221 2024-11-11T16:16:31,858 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:31,859 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#460 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:31,859 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/c790fc81b7c3430e9d4395f1fe7236dc is 50, key is test_row_0/B:col10/1731341790598/Put/seqid=0 2024-11-11T16:16:31,859 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/4e28187022224feaaf7d00bd25039546 is 50, key is test_row_0/A:col10/1731341790598/Put/seqid=0 2024-11-11T16:16:31,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742363_1539 (size=13119) 2024-11-11T16:16:31,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742362_1538 (size=13119) 2024-11-11T16:16:31,870 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/c790fc81b7c3430e9d4395f1fe7236dc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c790fc81b7c3430e9d4395f1fe7236dc 2024-11-11T16:16:31,874 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into c790fc81b7c3430e9d4395f1fe7236dc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:31,874 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:31,874 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=11, startTime=1731341791846; duration=0sec 2024-11-11T16:16:31,874 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:31,874 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:31,874 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T16:16:31,875 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T16:16:31,875 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:31,875 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,875 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/356a9fa87b9d426a894b33096d6b923d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a74d4e5f80ca489b909046e9f45833e3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/3ece60c6f2a543caba2a42b673009e46, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2e90c0ae84804a1b8bedcf2b9973a33f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/242967ee1cf545118326f25e2d002747] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=60.7 K 2024-11-11T16:16:31,876 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 356a9fa87b9d426a894b33096d6b923d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731341785232 2024-11-11T16:16:31,877 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a74d4e5f80ca489b909046e9f45833e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731341786478 2024-11-11T16:16:31,877 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ece60c6f2a543caba2a42b673009e46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731341787727 2024-11-11T16:16:31,878 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e90c0ae84804a1b8bedcf2b9973a33f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731341788976 2024-11-11T16:16:31,878 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 242967ee1cf545118326f25e2d002747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731341790270 2024-11-11T16:16:31,885 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:31,886 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/d8faa4ac950743a4b7da15db6dbcf437 is 50, key is test_row_0/C:col10/1731341790598/Put/seqid=0 2024-11-11T16:16:31,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742364_1540 (size=13119) 2024-11-11T16:16:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:31,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:16:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:31,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/d249c14837d241d7aad06cf3dffb55e2 is 50, key is test_row_0/A:col10/1731341790692/Put/seqid=0 2024-11-11T16:16:31,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742365_1541 (size=14741) 2024-11-11T16:16:31,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/d249c14837d241d7aad06cf3dffb55e2 2024-11-11T16:16:31,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/3c227201a64f4fc4b6f22041807379ac is 50, key is test_row_0/B:col10/1731341790692/Put/seqid=0 2024-11-11T16:16:31,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742366_1542 (size=12301) 2024-11-11T16:16:31,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/3c227201a64f4fc4b6f22041807379ac 2024-11-11T16:16:31,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9cc902d6968346be92ea86fd329bb1b8 is 50, key is test_row_0/C:col10/1731341790692/Put/seqid=0 2024-11-11T16:16:31,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742367_1543 (size=12301) 2024-11-11T16:16:31,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341851937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:31,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341851941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:31,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:31,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:31,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:31,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:32,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341852056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341852056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:32,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:32,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:32,267 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/4e28187022224feaaf7d00bd25039546 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/4e28187022224feaaf7d00bd25039546 2024-11-11T16:16:32,272 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into 4e28187022224feaaf7d00bd25039546(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:32,272 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:32,272 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=11, startTime=1731341791845; duration=0sec 2024-11-11T16:16:32,272 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:32,272 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341852276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341852276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:32,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:32,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:32,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:32,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:32,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:32,295 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/d8faa4ac950743a4b7da15db6dbcf437 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d8faa4ac950743a4b7da15db6dbcf437 2024-11-11T16:16:32,303 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into d8faa4ac950743a4b7da15db6dbcf437(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:32,303 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:32,303 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=11, startTime=1731341791846; duration=0sec 2024-11-11T16:16:32,303 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:32,303 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:32,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9cc902d6968346be92ea86fd329bb1b8 2024-11-11T16:16:32,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/d249c14837d241d7aad06cf3dffb55e2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d249c14837d241d7aad06cf3dffb55e2 2024-11-11T16:16:32,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d249c14837d241d7aad06cf3dffb55e2, entries=200, sequenceid=490, filesize=14.4 K 2024-11-11T16:16:32,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/3c227201a64f4fc4b6f22041807379ac as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3c227201a64f4fc4b6f22041807379ac 2024-11-11T16:16:32,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3c227201a64f4fc4b6f22041807379ac, entries=150, sequenceid=490, filesize=12.0 K 2024-11-11T16:16:32,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9cc902d6968346be92ea86fd329bb1b8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9cc902d6968346be92ea86fd329bb1b8 2024-11-11T16:16:32,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9cc902d6968346be92ea86fd329bb1b8, entries=150, sequenceid=490, filesize=12.0 K 2024-11-11T16:16:32,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9e768d9f517a9b5cf0bf527b36f5956b in 455ms, sequenceid=490, compaction requested=false 2024-11-11T16:16:32,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:32,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T16:16:32,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:32,445 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:16:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:32,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/54f0336ae70d444d8c5aba44a42a3d98 is 50, key is test_row_0/A:col10/1731341791939/Put/seqid=0 2024-11-11T16:16:32,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742368_1544 (size=12301) 2024-11-11T16:16:32,455 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/54f0336ae70d444d8c5aba44a42a3d98 2024-11-11T16:16:32,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e2026ead2b8472da14b8adf8a529b47 is 50, key is test_row_0/B:col10/1731341791939/Put/seqid=0 2024-11-11T16:16:32,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742369_1545 (size=12301) 2024-11-11T16:16:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:32,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:32,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341852717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341852717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T16:16:32,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341852835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341852846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:32,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:32,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:32,889 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e2026ead2b8472da14b8adf8a529b47 2024-11-11T16:16:32,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/2be3be5d41674ada9f60ecdcffb4f53c is 50, key is test_row_0/C:col10/1731341791939/Put/seqid=0 2024-11-11T16:16:32,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742370_1546 (size=12301) 2024-11-11T16:16:32,910 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/2be3be5d41674ada9f60ecdcffb4f53c 2024-11-11T16:16:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/54f0336ae70d444d8c5aba44a42a3d98 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/54f0336ae70d444d8c5aba44a42a3d98 2024-11-11T16:16:32,927 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/54f0336ae70d444d8c5aba44a42a3d98, entries=150, sequenceid=505, filesize=12.0 K 2024-11-11T16:16:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/2e2026ead2b8472da14b8adf8a529b47 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2026ead2b8472da14b8adf8a529b47 2024-11-11T16:16:32,969 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2026ead2b8472da14b8adf8a529b47, entries=150, sequenceid=505, filesize=12.0 K 2024-11-11T16:16:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/2be3be5d41674ada9f60ecdcffb4f53c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2be3be5d41674ada9f60ecdcffb4f53c 2024-11-11T16:16:32,975 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2be3be5d41674ada9f60ecdcffb4f53c, entries=150, sequenceid=505, filesize=12.0 K 2024-11-11T16:16:32,976 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9e768d9f517a9b5cf0bf527b36f5956b in 531ms, sequenceid=505, compaction requested=true 2024-11-11T16:16:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T16:16:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T16:16:32,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T16:16:32,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3890 sec 2024-11-11T16:16:32,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3930 sec 2024-11-11T16:16:33,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:16:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:33,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/83c0d2112a904918914cac47a13066dc is 50, key is test_row_0/A:col10/1731341792679/Put/seqid=0 2024-11-11T16:16:33,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742371_1547 (size=14741) 2024-11-11T16:16:33,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/83c0d2112a904918914cac47a13066dc 2024-11-11T16:16:33,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ac2fd2808787405791ea25c1e46d59dd is 50, key is test_row_0/B:col10/1731341792679/Put/seqid=0 2024-11-11T16:16:33,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742372_1548 (size=12301) 2024-11-11T16:16:33,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ac2fd2808787405791ea25c1e46d59dd 2024-11-11T16:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341853113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:33,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341853129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:33,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/0f2cb47bdeeb4bf18f2591730ad82f4d is 50, key is test_row_0/C:col10/1731341792679/Put/seqid=0 2024-11-11T16:16:33,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742373_1549 (size=12301) 2024-11-11T16:16:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341853236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:33,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341853256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:33,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341853445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:33,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341853466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:33,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/0f2cb47bdeeb4bf18f2591730ad82f4d 2024-11-11T16:16:33,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/83c0d2112a904918914cac47a13066dc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/83c0d2112a904918914cac47a13066dc 2024-11-11T16:16:33,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/83c0d2112a904918914cac47a13066dc, entries=200, sequenceid=525, filesize=14.4 K 2024-11-11T16:16:33,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/ac2fd2808787405791ea25c1e46d59dd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac2fd2808787405791ea25c1e46d59dd 2024-11-11T16:16:33,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac2fd2808787405791ea25c1e46d59dd, entries=150, sequenceid=525, filesize=12.0 K 2024-11-11T16:16:33,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/0f2cb47bdeeb4bf18f2591730ad82f4d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/0f2cb47bdeeb4bf18f2591730ad82f4d 2024-11-11T16:16:33,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/0f2cb47bdeeb4bf18f2591730ad82f4d, entries=150, sequenceid=525, filesize=12.0 K 2024-11-11T16:16:33,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9e768d9f517a9b5cf0bf527b36f5956b in 530ms, sequenceid=525, compaction requested=true 2024-11-11T16:16:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:33,595 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:33,595 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:33,596 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:33,596 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:33,596 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:33,596 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:33,596 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:33,596 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:33,596 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/4e28187022224feaaf7d00bd25039546, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d249c14837d241d7aad06cf3dffb55e2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/54f0336ae70d444d8c5aba44a42a3d98, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/83c0d2112a904918914cac47a13066dc] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=53.6 K 2024-11-11T16:16:33,596 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c790fc81b7c3430e9d4395f1fe7236dc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3c227201a64f4fc4b6f22041807379ac, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2026ead2b8472da14b8adf8a529b47, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac2fd2808787405791ea25c1e46d59dd] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=48.8 K 2024-11-11T16:16:33,597 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e28187022224feaaf7d00bd25039546, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731341790270 2024-11-11T16:16:33,597 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c790fc81b7c3430e9d4395f1fe7236dc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731341790270 2024-11-11T16:16:33,597 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d249c14837d241d7aad06cf3dffb55e2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731341790666 2024-11-11T16:16:33,597 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c227201a64f4fc4b6f22041807379ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731341790666 2024-11-11T16:16:33,598 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f0336ae70d444d8c5aba44a42a3d98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731341791920 2024-11-11T16:16:33,598 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2026ead2b8472da14b8adf8a529b47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731341791920 2024-11-11T16:16:33,598 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83c0d2112a904918914cac47a13066dc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731341792679 2024-11-11T16:16:33,598 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2fd2808787405791ea25c1e46d59dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731341792679 2024-11-11T16:16:33,607 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:33,608 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/23e5a833b27245eeb3f6cc89792ce874 is 50, key is test_row_0/B:col10/1731341792679/Put/seqid=0 2024-11-11T16:16:33,611 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:33,612 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f859796f78e845bd9e6a14ed49d43732 is 50, key is test_row_0/A:col10/1731341792679/Put/seqid=0 2024-11-11T16:16:33,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742374_1550 (size=13255) 2024-11-11T16:16:33,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742375_1551 (size=13255) 2024-11-11T16:16:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:33,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:16:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:33,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f8be8e0226014e019d0dd227c235d538 is 50, key is test_row_0/A:col10/1731341793760/Put/seqid=0 2024-11-11T16:16:33,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742376_1552 (size=14741) 2024-11-11T16:16:33,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341853896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:33,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341853909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,022 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/23e5a833b27245eeb3f6cc89792ce874 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/23e5a833b27245eeb3f6cc89792ce874 2024-11-11T16:16:34,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341854016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,027 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into 23e5a833b27245eeb3f6cc89792ce874(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:34,027 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:34,028 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=12, startTime=1731341793595; duration=0sec 2024-11-11T16:16:34,028 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:34,028 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:34,028 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:34,031 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:34,031 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:34,031 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:34,031 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d8faa4ac950743a4b7da15db6dbcf437, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9cc902d6968346be92ea86fd329bb1b8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2be3be5d41674ada9f60ecdcffb4f53c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/0f2cb47bdeeb4bf18f2591730ad82f4d] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=48.8 K 2024-11-11T16:16:34,031 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d8faa4ac950743a4b7da15db6dbcf437, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731341790270 2024-11-11T16:16:34,032 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f859796f78e845bd9e6a14ed49d43732 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f859796f78e845bd9e6a14ed49d43732 2024-11-11T16:16:34,032 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc902d6968346be92ea86fd329bb1b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731341790666 2024-11-11T16:16:34,032 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be3be5d41674ada9f60ecdcffb4f53c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731341791920 2024-11-11T16:16:34,033 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f2cb47bdeeb4bf18f2591730ad82f4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731341792679 2024-11-11T16:16:34,037 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into f859796f78e845bd9e6a14ed49d43732(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:34,037 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:34,037 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=12, startTime=1731341793595; duration=0sec 2024-11-11T16:16:34,038 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:34,038 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341854036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,048 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,049 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/91d93bf1bd03482a96ac02267d40fe82 is 50, key is test_row_0/C:col10/1731341792679/Put/seqid=0 2024-11-11T16:16:34,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742377_1553 (size=13255) 2024-11-11T16:16:34,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f8be8e0226014e019d0dd227c235d538 2024-11-11T16:16:34,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/18b0714ed8d04a6f948fe23b680d6e19 is 50, key is test_row_0/B:col10/1731341793760/Put/seqid=0 2024-11-11T16:16:34,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742378_1554 (size=12301) 2024-11-11T16:16:34,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/18b0714ed8d04a6f948fe23b680d6e19 2024-11-11T16:16:34,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/cee56a599e5c41b48ed77919b26d19df is 50, key is test_row_0/C:col10/1731341793760/Put/seqid=0 2024-11-11T16:16:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341854236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742379_1555 (size=12301) 2024-11-11T16:16:34,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/cee56a599e5c41b48ed77919b26d19df 2024-11-11T16:16:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341854256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/f8be8e0226014e019d0dd227c235d538 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f8be8e0226014e019d0dd227c235d538 2024-11-11T16:16:34,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f8be8e0226014e019d0dd227c235d538, entries=200, sequenceid=538, filesize=14.4 K 2024-11-11T16:16:34,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/18b0714ed8d04a6f948fe23b680d6e19 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/18b0714ed8d04a6f948fe23b680d6e19 2024-11-11T16:16:34,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/18b0714ed8d04a6f948fe23b680d6e19, entries=150, sequenceid=538, filesize=12.0 K 2024-11-11T16:16:34,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/cee56a599e5c41b48ed77919b26d19df as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/cee56a599e5c41b48ed77919b26d19df 2024-11-11T16:16:34,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/cee56a599e5c41b48ed77919b26d19df, entries=150, sequenceid=538, filesize=12.0 K 2024-11-11T16:16:34,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9e768d9f517a9b5cf0bf527b36f5956b in 547ms, sequenceid=538, compaction requested=false 2024-11-11T16:16:34,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:34,477 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/91d93bf1bd03482a96ac02267d40fe82 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/91d93bf1bd03482a96ac02267d40fe82 2024-11-11T16:16:34,481 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 91d93bf1bd03482a96ac02267d40fe82(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:34,481 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:34,481 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=12, startTime=1731341793595; duration=0sec 2024-11-11T16:16:34,481 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:34,481 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:34,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T16:16:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:34,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/0a0cf039e23d4324a5329137d0b154dd is 50, key is test_row_0/A:col10/1731341794572/Put/seqid=0 2024-11-11T16:16:34,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742380_1556 (size=14741) 2024-11-11T16:16:34,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341854617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341854621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341854725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341854736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T16:16:34,757 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:34,757 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T16:16:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T16:16:34,759 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:34,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:34,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T16:16:34,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T16:16:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:34,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:34,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341854947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341854959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:34,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/0a0cf039e23d4324a5329137d0b154dd 2024-11-11T16:16:34,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/1644612eb93544eb848b731752136d38 is 50, key is test_row_0/B:col10/1731341794572/Put/seqid=0 2024-11-11T16:16:34,997 INFO [Thread-2092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2092. 2024-11-11T16:16:34,997 DEBUG [Thread-2092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:34,998 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:34,998 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:34,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:34,998 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-11-11T16:16:34,998 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:34,998 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:34,998 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:34,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:35,006 INFO [Thread-2088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2088. 2024-11-11T16:16:35,007 DEBUG [Thread-2088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:35,007 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:35,007 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:35,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:35,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742381_1557 (size=12301) 2024-11-11T16:16:35,009 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-11-11T16:16:35,009 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:35,009 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:35,009 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:35,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:35,010 INFO [Thread-2090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2090. 2024-11-11T16:16:35,010 DEBUG [Thread-2090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:35,010 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:35,010 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:35,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:35,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T16:16:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T16:16:35,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T16:16:35,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341855265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341855276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:35,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T16:16:35,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:35,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T16:16:35,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/1644612eb93544eb848b731752136d38 2024-11-11T16:16:35,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/6df08ac1027849e09f3e82b2b23a2b81 is 50, key is test_row_0/C:col10/1731341794572/Put/seqid=0 2024-11-11T16:16:35,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742382_1558 (size=12301) 2024-11-11T16:16:35,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T16:16:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T16:16:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. as already flushing 2024-11-11T16:16:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:35,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33466 deadline: 1731341855775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:35,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:35,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33458 deadline: 1731341855786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 , the old value is region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., hostname=16b413a53992,45121,1731341648557, seqNum=2 because the exception is null or not the one we care about 2024-11-11T16:16:35,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/6df08ac1027849e09f3e82b2b23a2b81 2024-11-11T16:16:35,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/0a0cf039e23d4324a5329137d0b154dd as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/0a0cf039e23d4324a5329137d0b154dd 2024-11-11T16:16:35,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/0a0cf039e23d4324a5329137d0b154dd, entries=200, sequenceid=561, filesize=14.4 K 2024-11-11T16:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/1644612eb93544eb848b731752136d38 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1644612eb93544eb848b731752136d38 2024-11-11T16:16:35,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1644612eb93544eb848b731752136d38, entries=150, sequenceid=561, filesize=12.0 K 2024-11-11T16:16:35,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/6df08ac1027849e09f3e82b2b23a2b81 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/6df08ac1027849e09f3e82b2b23a2b81 2024-11-11T16:16:35,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/6df08ac1027849e09f3e82b2b23a2b81, entries=150, sequenceid=561, filesize=12.0 K 2024-11-11T16:16:35,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9e768d9f517a9b5cf0bf527b36f5956b in 1258ms, sequenceid=561, compaction requested=true 2024-11-11T16:16:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:35,832 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e768d9f517a9b5cf0bf527b36f5956b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:35,833 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:35,833 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:35,833 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:35,833 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/B is initiating minor compaction (all files) 2024-11-11T16:16:35,833 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/A is initiating minor compaction (all files) 2024-11-11T16:16:35,833 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/B in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,833 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/A in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,833 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/23e5a833b27245eeb3f6cc89792ce874, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/18b0714ed8d04a6f948fe23b680d6e19, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1644612eb93544eb848b731752136d38] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=37.0 K 2024-11-11T16:16:35,833 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f859796f78e845bd9e6a14ed49d43732, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f8be8e0226014e019d0dd227c235d538, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/0a0cf039e23d4324a5329137d0b154dd] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=41.7 K 2024-11-11T16:16:35,834 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f859796f78e845bd9e6a14ed49d43732, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731341792679 2024-11-11T16:16:35,834 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e5a833b27245eeb3f6cc89792ce874, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731341792679 2024-11-11T16:16:35,834 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b0714ed8d04a6f948fe23b680d6e19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1731341793076 2024-11-11T16:16:35,834 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8be8e0226014e019d0dd227c235d538, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1731341793076 2024-11-11T16:16:35,835 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a0cf039e23d4324a5329137d0b154dd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1731341793877 2024-11-11T16:16:35,835 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 1644612eb93544eb848b731752136d38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1731341793877 2024-11-11T16:16:35,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T16:16:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:35,836 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T16:16:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:35,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/44c8d16bda7941c19c8b3db518493762 is 50, key is test_row_0/A:col10/1731341794616/Put/seqid=0 2024-11-11T16:16:35,842 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#A#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:35,843 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/885df65ef9a14480b50be9bc506d21d9 is 50, key is test_row_0/A:col10/1731341794572/Put/seqid=0 2024-11-11T16:16:35,843 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:35,843 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/43bb7cd4ec5e4327a87155c115d937c6 is 50, key is test_row_0/B:col10/1731341794572/Put/seqid=0 2024-11-11T16:16:35,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742383_1559 (size=12301) 2024-11-11T16:16:35,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742384_1560 (size=13357) 2024-11-11T16:16:35,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742385_1561 (size=13357) 2024-11-11T16:16:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T16:16:36,249 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/44c8d16bda7941c19c8b3db518493762 2024-11-11T16:16:36,253 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/885df65ef9a14480b50be9bc506d21d9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/885df65ef9a14480b50be9bc506d21d9 2024-11-11T16:16:36,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/749c32859ed645aba15d2ff7aa00849f is 50, key is test_row_0/B:col10/1731341794616/Put/seqid=0 2024-11-11T16:16:36,258 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/A of 9e768d9f517a9b5cf0bf527b36f5956b into 885df65ef9a14480b50be9bc506d21d9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:36,258 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:36,258 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/A, priority=13, startTime=1731341795832; duration=0sec 2024-11-11T16:16:36,258 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:36,258 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:A 2024-11-11T16:16:36,258 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:36,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742386_1562 (size=12301) 2024-11-11T16:16:36,259 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/43bb7cd4ec5e4327a87155c115d937c6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/43bb7cd4ec5e4327a87155c115d937c6 2024-11-11T16:16:36,259 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/749c32859ed645aba15d2ff7aa00849f 2024-11-11T16:16:36,259 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:36,259 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 9e768d9f517a9b5cf0bf527b36f5956b/C is initiating minor compaction (all files) 2024-11-11T16:16:36,259 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e768d9f517a9b5cf0bf527b36f5956b/C in TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:36,260 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/91d93bf1bd03482a96ac02267d40fe82, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/cee56a599e5c41b48ed77919b26d19df, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/6df08ac1027849e09f3e82b2b23a2b81] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp, totalSize=37.0 K 2024-11-11T16:16:36,260 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d93bf1bd03482a96ac02267d40fe82, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731341792679 2024-11-11T16:16:36,260 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cee56a599e5c41b48ed77919b26d19df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1731341793076 2024-11-11T16:16:36,261 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df08ac1027849e09f3e82b2b23a2b81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1731341793877 2024-11-11T16:16:36,263 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/B of 9e768d9f517a9b5cf0bf527b36f5956b into 43bb7cd4ec5e4327a87155c115d937c6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:36,263 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:36,263 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/B, priority=13, startTime=1731341795832; duration=0sec 2024-11-11T16:16:36,263 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:36,263 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:B 2024-11-11T16:16:36,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9e248340c92547d99f4141e0d85ea578 is 50, key is test_row_0/C:col10/1731341794616/Put/seqid=0 2024-11-11T16:16:36,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742387_1563 (size=12301) 2024-11-11T16:16:36,269 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e768d9f517a9b5cf0bf527b36f5956b#C#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:36,270 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/647c33b47c3c4c078cd5ac945242bce7 is 50, key is test_row_0/C:col10/1731341794572/Put/seqid=0 2024-11-11T16:16:36,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742388_1564 (size=13357) 2024-11-11T16:16:36,365 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T16:16:36,669 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9e248340c92547d99f4141e0d85ea578 2024-11-11T16:16:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/44c8d16bda7941c19c8b3db518493762 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/44c8d16bda7941c19c8b3db518493762 2024-11-11T16:16:36,677 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/44c8d16bda7941c19c8b3db518493762, entries=150, sequenceid=573, filesize=12.0 K 2024-11-11T16:16:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/749c32859ed645aba15d2ff7aa00849f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/749c32859ed645aba15d2ff7aa00849f 2024-11-11T16:16:36,679 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/647c33b47c3c4c078cd5ac945242bce7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/647c33b47c3c4c078cd5ac945242bce7 2024-11-11T16:16:36,680 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/749c32859ed645aba15d2ff7aa00849f, entries=150, sequenceid=573, filesize=12.0 K 2024-11-11T16:16:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9e248340c92547d99f4141e0d85ea578 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9e248340c92547d99f4141e0d85ea578 2024-11-11T16:16:36,684 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e768d9f517a9b5cf0bf527b36f5956b/C of 9e768d9f517a9b5cf0bf527b36f5956b into 647c33b47c3c4c078cd5ac945242bce7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:36,684 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:36,684 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., storeName=9e768d9f517a9b5cf0bf527b36f5956b/C, priority=13, startTime=1731341795833; duration=0sec 2024-11-11T16:16:36,684 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:36,684 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e768d9f517a9b5cf0bf527b36f5956b:C 2024-11-11T16:16:36,685 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9e248340c92547d99f4141e0d85ea578, entries=150, sequenceid=573, filesize=12.0 K 2024-11-11T16:16:36,686 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 9e768d9f517a9b5cf0bf527b36f5956b in 850ms, sequenceid=573, compaction requested=false 2024-11-11T16:16:36,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 9e768d9f517a9b5cf0bf527b36f5956b: 2024-11-11T16:16:36,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:36,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T16:16:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T16:16:36,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T16:16:36,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9280 sec 2024-11-11T16:16:36,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9340 sec 2024-11-11T16:16:36,788 INFO [Thread-2081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2081. 2024-11-11T16:16:36,788 DEBUG [Thread-2081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:36,788 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:36,788 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:36,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:36,797 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-11-11T16:16:36,797 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:36,797 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:36,797 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:36,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T16:16:36,916 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:38,217 INFO [Thread-2079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2079. 2024-11-11T16:16:38,217 DEBUG [Thread-2079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:38,217 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,217 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:38,247 INFO [Thread-2083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2083. 2024-11-11T16:16:38,247 DEBUG [Thread-2083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:38,247 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,248 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:38,277 INFO [Thread-2077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2077. 2024-11-11T16:16:38,277 DEBUG [Thread-2077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:16:38,277 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,277 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 176 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 198 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1003 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2999 rows 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1007 2024-11-11T16:16:38,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3012 rows 2024-11-11T16:16:38,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1006 2024-11-11T16:16:38,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3017 rows 2024-11-11T16:16:38,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1010 2024-11-11T16:16:38,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3019 rows 2024-11-11T16:16:38,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1008 2024-11-11T16:16:38,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3015 rows 2024-11-11T16:16:38,278 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:16:38,278 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:16:38,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:38,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:16:38,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T16:16:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T16:16:38,281 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341798281"}]},"ts":"1731341798281"} 2024-11-11T16:16:38,282 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T16:16:38,282 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T16:16:38,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:16:38,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, UNASSIGN}] 2024-11-11T16:16:38,285 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, UNASSIGN 2024-11-11T16:16:38,285 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=9e768d9f517a9b5cf0bf527b36f5956b, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:38,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, UNASSIGN because future has completed 2024-11-11T16:16:38,287 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:16:38,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:16:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T16:16:38,439 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:38,439 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:16:38,439 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 9e768d9f517a9b5cf0bf527b36f5956b, disabling compactions & flushes 2024-11-11T16:16:38,439 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:38,439 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:38,439 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. after waiting 0 ms 2024-11-11T16:16:38,439 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:38,439 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 9e768d9f517a9b5cf0bf527b36f5956b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T16:16:38,440 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=A 2024-11-11T16:16:38,440 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:38,440 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=B 2024-11-11T16:16:38,440 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:38,440 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e768d9f517a9b5cf0bf527b36f5956b, store=C 2024-11-11T16:16:38,440 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:38,445 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/aade8c93c7f14139aaf412585ed704aa is 50, key is test_row_0/A:col10/1731341796796/Put/seqid=0 2024-11-11T16:16:38,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742389_1565 (size=12301) 2024-11-11T16:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T16:16:38,850 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/aade8c93c7f14139aaf412585ed704aa 2024-11-11T16:16:38,855 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/1b7358de43684e17b2c8d4c255c45969 is 50, key is test_row_0/B:col10/1731341796796/Put/seqid=0 2024-11-11T16:16:38,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742390_1566 (size=12301) 2024-11-11T16:16:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T16:16:39,259 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/1b7358de43684e17b2c8d4c255c45969 2024-11-11T16:16:39,264 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9d473448367e4a64a36466984e6dade7 is 50, key is test_row_0/C:col10/1731341796796/Put/seqid=0 2024-11-11T16:16:39,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742391_1567 (size=12301) 2024-11-11T16:16:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T16:16:39,668 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9d473448367e4a64a36466984e6dade7 2024-11-11T16:16:39,672 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/A/aade8c93c7f14139aaf412585ed704aa as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/aade8c93c7f14139aaf412585ed704aa 2024-11-11T16:16:39,675 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/aade8c93c7f14139aaf412585ed704aa, entries=150, sequenceid=584, filesize=12.0 K 2024-11-11T16:16:39,676 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/B/1b7358de43684e17b2c8d4c255c45969 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1b7358de43684e17b2c8d4c255c45969 2024-11-11T16:16:39,679 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1b7358de43684e17b2c8d4c255c45969, entries=150, sequenceid=584, filesize=12.0 K 2024-11-11T16:16:39,680 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/.tmp/C/9d473448367e4a64a36466984e6dade7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d473448367e4a64a36466984e6dade7 2024-11-11T16:16:39,683 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d473448367e4a64a36466984e6dade7, entries=150, sequenceid=584, filesize=12.0 K 2024-11-11T16:16:39,684 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9e768d9f517a9b5cf0bf527b36f5956b in 1244ms, sequenceid=584, compaction requested=true 2024-11-11T16:16:39,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/301bdf57e33f43e085ee10e76f230172, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/64213a7b7a1640738d09a5f672ed3330, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eea469229b3d480db89f9976ea561378, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/03d9d4332f734ebda612eb1010c73fd8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b65d63904c4945c89d1adf88b22aedfe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/42405786a66440ae91aa79d51a6f9b63, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f59f06a332454bd5a99e9288c4f0a849, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/48dc1dbcf4004914bd04a238d411572c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eaa64707151e4ea6b6d809b7d9447571, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/a29e3e9186284acc8592ba9643cffab4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/fc89903021c149de944da16790ee3a27, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/972c68f87bd94962a7cfc1891d5732ab, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b9b161b88b2846ea8554fbb7705030c1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/acd47fb048ce44e89bf4434503708cc6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/18357dbd70b5436397bbaf1419bba73c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/62b08659ad2442dc9c79e5a2e4c2e2d6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/82b572c2329e4bfbba7a5e869b098ade, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/05ada2d6b79e4618a4cf58705c944890, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/e3936543ab7e4fc28aeed876779c66a5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/31c873bee8a04e4ab2822c9cb03de47f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b391760faeb74e0294a678297929ef84, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/ab912b4b6c03454d9ef5301efbc2c707, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d1d5823fd14c455397ffabf344b45ef2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/dd03f19440d04dfc8e3f0ee8940e397d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/4e28187022224feaaf7d00bd25039546, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d249c14837d241d7aad06cf3dffb55e2, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/54f0336ae70d444d8c5aba44a42a3d98, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/83c0d2112a904918914cac47a13066dc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f859796f78e845bd9e6a14ed49d43732, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f8be8e0226014e019d0dd227c235d538, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/0a0cf039e23d4324a5329137d0b154dd] to archive 2024-11-11T16:16:39,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:16:39,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/301bdf57e33f43e085ee10e76f230172 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/301bdf57e33f43e085ee10e76f230172 2024-11-11T16:16:39,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/64213a7b7a1640738d09a5f672ed3330 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/64213a7b7a1640738d09a5f672ed3330 2024-11-11T16:16:39,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eea469229b3d480db89f9976ea561378 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eea469229b3d480db89f9976ea561378 2024-11-11T16:16:39,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/03d9d4332f734ebda612eb1010c73fd8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/03d9d4332f734ebda612eb1010c73fd8 2024-11-11T16:16:39,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b65d63904c4945c89d1adf88b22aedfe to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b65d63904c4945c89d1adf88b22aedfe 2024-11-11T16:16:39,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/42405786a66440ae91aa79d51a6f9b63 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/42405786a66440ae91aa79d51a6f9b63 2024-11-11T16:16:39,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f59f06a332454bd5a99e9288c4f0a849 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f59f06a332454bd5a99e9288c4f0a849 2024-11-11T16:16:39,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/48dc1dbcf4004914bd04a238d411572c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/48dc1dbcf4004914bd04a238d411572c 2024-11-11T16:16:39,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eaa64707151e4ea6b6d809b7d9447571 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/eaa64707151e4ea6b6d809b7d9447571 2024-11-11T16:16:39,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/a29e3e9186284acc8592ba9643cffab4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/a29e3e9186284acc8592ba9643cffab4 2024-11-11T16:16:39,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/fc89903021c149de944da16790ee3a27 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/fc89903021c149de944da16790ee3a27 2024-11-11T16:16:39,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/972c68f87bd94962a7cfc1891d5732ab to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/972c68f87bd94962a7cfc1891d5732ab 2024-11-11T16:16:39,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b9b161b88b2846ea8554fbb7705030c1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b9b161b88b2846ea8554fbb7705030c1 2024-11-11T16:16:39,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/acd47fb048ce44e89bf4434503708cc6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/acd47fb048ce44e89bf4434503708cc6 2024-11-11T16:16:39,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/18357dbd70b5436397bbaf1419bba73c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/18357dbd70b5436397bbaf1419bba73c 2024-11-11T16:16:39,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/62b08659ad2442dc9c79e5a2e4c2e2d6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/62b08659ad2442dc9c79e5a2e4c2e2d6 2024-11-11T16:16:39,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/82b572c2329e4bfbba7a5e869b098ade to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/82b572c2329e4bfbba7a5e869b098ade 2024-11-11T16:16:39,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/05ada2d6b79e4618a4cf58705c944890 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/05ada2d6b79e4618a4cf58705c944890 2024-11-11T16:16:39,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/e3936543ab7e4fc28aeed876779c66a5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/e3936543ab7e4fc28aeed876779c66a5 2024-11-11T16:16:39,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/31c873bee8a04e4ab2822c9cb03de47f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/31c873bee8a04e4ab2822c9cb03de47f 2024-11-11T16:16:39,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b391760faeb74e0294a678297929ef84 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/b391760faeb74e0294a678297929ef84 2024-11-11T16:16:39,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/ab912b4b6c03454d9ef5301efbc2c707 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/ab912b4b6c03454d9ef5301efbc2c707 2024-11-11T16:16:39,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d1d5823fd14c455397ffabf344b45ef2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d1d5823fd14c455397ffabf344b45ef2 2024-11-11T16:16:39,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/dd03f19440d04dfc8e3f0ee8940e397d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/dd03f19440d04dfc8e3f0ee8940e397d 2024-11-11T16:16:39,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/4e28187022224feaaf7d00bd25039546 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/4e28187022224feaaf7d00bd25039546 2024-11-11T16:16:39,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d249c14837d241d7aad06cf3dffb55e2 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/d249c14837d241d7aad06cf3dffb55e2 2024-11-11T16:16:39,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/54f0336ae70d444d8c5aba44a42a3d98 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/54f0336ae70d444d8c5aba44a42a3d98 2024-11-11T16:16:39,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/83c0d2112a904918914cac47a13066dc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/83c0d2112a904918914cac47a13066dc 2024-11-11T16:16:39,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f859796f78e845bd9e6a14ed49d43732 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f859796f78e845bd9e6a14ed49d43732 2024-11-11T16:16:39,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f8be8e0226014e019d0dd227c235d538 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/f8be8e0226014e019d0dd227c235d538 2024-11-11T16:16:39,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/0a0cf039e23d4324a5329137d0b154dd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/0a0cf039e23d4324a5329137d0b154dd 2024-11-11T16:16:39,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b6a08f0af9ea417694f48e0f730abae4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e1be90577dc40f4b5ad3a15f1335e79, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/d66877a739524e6cb41bbc4e9a2a301b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3f2bdf0025ff4a3d8f82891be2d2ebc1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/882c490b24da4be1ba6cb71175e1c1e5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5ae5390c930d43ba810d4a387addbb48, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac9ec8d1a8e7410fb1e9f3ea443337d1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/37baae1348c441fd9bdadc5b9fd04d29, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/52f2b65cbd3d42929f3126d44433a798, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c9fdfa88326646c6a945cae45ed9953e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/70a5cc9ad46d4a7db375b52ef86e429b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5dc6ff9239c14135a4f1efb47afa1261, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/976cefec078c49e0a69e02c35cc1d21e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2fe792e67b409d8ae4dd60b9efcfe9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b4285c416ecb414bb32aa03cbc2e854d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5f83a823d610490c91ca9f51ce15958e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/92cb0da5ca724faa8b87b64bd836533f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/979bd7bba44749b984dc9c12331abc29, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ae1d8a9f24b54833aaf2f3da4c732d80, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/e256bd9bbc03438b8c61dc2185ae6c62, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/4ae19636bebe4348a536f799ab515212, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/180c123a0bf940dea4b3060d60ad6ecb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/62ae2f10ab4f4beeb4e78e35cb8a86b1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c790fc81b7c3430e9d4395f1fe7236dc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/7ef200c2d52742bd86c6079467aa09de, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3c227201a64f4fc4b6f22041807379ac, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2026ead2b8472da14b8adf8a529b47, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/23e5a833b27245eeb3f6cc89792ce874, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac2fd2808787405791ea25c1e46d59dd, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/18b0714ed8d04a6f948fe23b680d6e19, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1644612eb93544eb848b731752136d38] to archive 2024-11-11T16:16:39,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:16:39,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b6a08f0af9ea417694f48e0f730abae4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b6a08f0af9ea417694f48e0f730abae4 2024-11-11T16:16:39,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e1be90577dc40f4b5ad3a15f1335e79 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e1be90577dc40f4b5ad3a15f1335e79 2024-11-11T16:16:39,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/d66877a739524e6cb41bbc4e9a2a301b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/d66877a739524e6cb41bbc4e9a2a301b 2024-11-11T16:16:39,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3f2bdf0025ff4a3d8f82891be2d2ebc1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3f2bdf0025ff4a3d8f82891be2d2ebc1 2024-11-11T16:16:39,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/882c490b24da4be1ba6cb71175e1c1e5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/882c490b24da4be1ba6cb71175e1c1e5 2024-11-11T16:16:39,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5ae5390c930d43ba810d4a387addbb48 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5ae5390c930d43ba810d4a387addbb48 2024-11-11T16:16:39,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac9ec8d1a8e7410fb1e9f3ea443337d1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac9ec8d1a8e7410fb1e9f3ea443337d1 2024-11-11T16:16:39,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/37baae1348c441fd9bdadc5b9fd04d29 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/37baae1348c441fd9bdadc5b9fd04d29 2024-11-11T16:16:39,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/52f2b65cbd3d42929f3126d44433a798 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/52f2b65cbd3d42929f3126d44433a798 2024-11-11T16:16:39,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c9fdfa88326646c6a945cae45ed9953e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c9fdfa88326646c6a945cae45ed9953e 2024-11-11T16:16:39,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/70a5cc9ad46d4a7db375b52ef86e429b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/70a5cc9ad46d4a7db375b52ef86e429b 2024-11-11T16:16:39,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5dc6ff9239c14135a4f1efb47afa1261 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5dc6ff9239c14135a4f1efb47afa1261 2024-11-11T16:16:39,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/976cefec078c49e0a69e02c35cc1d21e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/976cefec078c49e0a69e02c35cc1d21e 2024-11-11T16:16:39,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2fe792e67b409d8ae4dd60b9efcfe9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2fe792e67b409d8ae4dd60b9efcfe9 2024-11-11T16:16:39,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b4285c416ecb414bb32aa03cbc2e854d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/b4285c416ecb414bb32aa03cbc2e854d 2024-11-11T16:16:39,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5f83a823d610490c91ca9f51ce15958e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/5f83a823d610490c91ca9f51ce15958e 2024-11-11T16:16:39,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/92cb0da5ca724faa8b87b64bd836533f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/92cb0da5ca724faa8b87b64bd836533f 2024-11-11T16:16:39,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/979bd7bba44749b984dc9c12331abc29 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/979bd7bba44749b984dc9c12331abc29 2024-11-11T16:16:39,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ae1d8a9f24b54833aaf2f3da4c732d80 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ae1d8a9f24b54833aaf2f3da4c732d80 2024-11-11T16:16:39,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/e256bd9bbc03438b8c61dc2185ae6c62 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/e256bd9bbc03438b8c61dc2185ae6c62 2024-11-11T16:16:39,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/4ae19636bebe4348a536f799ab515212 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/4ae19636bebe4348a536f799ab515212 2024-11-11T16:16:39,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/180c123a0bf940dea4b3060d60ad6ecb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/180c123a0bf940dea4b3060d60ad6ecb 2024-11-11T16:16:39,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/62ae2f10ab4f4beeb4e78e35cb8a86b1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/62ae2f10ab4f4beeb4e78e35cb8a86b1 2024-11-11T16:16:39,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c790fc81b7c3430e9d4395f1fe7236dc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/c790fc81b7c3430e9d4395f1fe7236dc 2024-11-11T16:16:39,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/7ef200c2d52742bd86c6079467aa09de to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/7ef200c2d52742bd86c6079467aa09de 2024-11-11T16:16:39,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3c227201a64f4fc4b6f22041807379ac to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/3c227201a64f4fc4b6f22041807379ac 2024-11-11T16:16:39,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2026ead2b8472da14b8adf8a529b47 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/2e2026ead2b8472da14b8adf8a529b47 2024-11-11T16:16:39,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/23e5a833b27245eeb3f6cc89792ce874 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/23e5a833b27245eeb3f6cc89792ce874 2024-11-11T16:16:39,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac2fd2808787405791ea25c1e46d59dd to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/ac2fd2808787405791ea25c1e46d59dd 2024-11-11T16:16:39,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/18b0714ed8d04a6f948fe23b680d6e19 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/18b0714ed8d04a6f948fe23b680d6e19 2024-11-11T16:16:39,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1644612eb93544eb848b731752136d38 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1644612eb93544eb848b731752136d38 2024-11-11T16:16:39,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/664adad8747d44e7b9402c230ff7725e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/8bb547c48b4a423eb205d2414dc393c0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/23567a779c424b578cf1d0ce781da5bf, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/264687af98b1443fbcb408338a11eaf7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/01d8fad6ef904a8ba8248c398ddcb267, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/ee5137b2936c46658a08512d749a502b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9c8f8fd841764733bbf74f5ccebbd20b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d51d85d60c564fd5aef8b636814f4f6b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/782002e66b844a03a203c3b705ac3764, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/5035462a96d442f5ae1a3af6f8d74c3f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4e23da5cbc954e9588ef1d1262899fb9, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a3269f987cff436db86e7adde1af04b1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/145112af48924b319ee6c2af44beb310, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/f741fe13a71c4b0b8c693da3ef5ff052, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d908c8a2bb746599a47106b5dcdd6fe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/013a170920c24917b7261aae9da29920, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/79b1f6dcd1404582a39723f98c0dd6bc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4af36e1f356e468992aea45e71c79fb5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/356a9fa87b9d426a894b33096d6b923d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/633d2a4c87454db08f5eb3a9aece8d5b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a74d4e5f80ca489b909046e9f45833e3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/3ece60c6f2a543caba2a42b673009e46, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2e90c0ae84804a1b8bedcf2b9973a33f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d8faa4ac950743a4b7da15db6dbcf437, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/242967ee1cf545118326f25e2d002747, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9cc902d6968346be92ea86fd329bb1b8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2be3be5d41674ada9f60ecdcffb4f53c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/91d93bf1bd03482a96ac02267d40fe82, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/0f2cb47bdeeb4bf18f2591730ad82f4d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/cee56a599e5c41b48ed77919b26d19df, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/6df08ac1027849e09f3e82b2b23a2b81] to archive 2024-11-11T16:16:39,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:16:39,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/664adad8747d44e7b9402c230ff7725e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/664adad8747d44e7b9402c230ff7725e 2024-11-11T16:16:39,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/8bb547c48b4a423eb205d2414dc393c0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/8bb547c48b4a423eb205d2414dc393c0 2024-11-11T16:16:39,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/23567a779c424b578cf1d0ce781da5bf to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/23567a779c424b578cf1d0ce781da5bf 2024-11-11T16:16:39,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/264687af98b1443fbcb408338a11eaf7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/264687af98b1443fbcb408338a11eaf7 2024-11-11T16:16:39,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/01d8fad6ef904a8ba8248c398ddcb267 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/01d8fad6ef904a8ba8248c398ddcb267 2024-11-11T16:16:39,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/ee5137b2936c46658a08512d749a502b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/ee5137b2936c46658a08512d749a502b 2024-11-11T16:16:39,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9c8f8fd841764733bbf74f5ccebbd20b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9c8f8fd841764733bbf74f5ccebbd20b 2024-11-11T16:16:39,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d51d85d60c564fd5aef8b636814f4f6b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d51d85d60c564fd5aef8b636814f4f6b 2024-11-11T16:16:39,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/782002e66b844a03a203c3b705ac3764 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/782002e66b844a03a203c3b705ac3764 2024-11-11T16:16:39,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/5035462a96d442f5ae1a3af6f8d74c3f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/5035462a96d442f5ae1a3af6f8d74c3f 2024-11-11T16:16:39,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4e23da5cbc954e9588ef1d1262899fb9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4e23da5cbc954e9588ef1d1262899fb9 2024-11-11T16:16:39,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a3269f987cff436db86e7adde1af04b1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a3269f987cff436db86e7adde1af04b1 2024-11-11T16:16:39,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/145112af48924b319ee6c2af44beb310 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/145112af48924b319ee6c2af44beb310 2024-11-11T16:16:39,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/f741fe13a71c4b0b8c693da3ef5ff052 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/f741fe13a71c4b0b8c693da3ef5ff052 2024-11-11T16:16:39,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d908c8a2bb746599a47106b5dcdd6fe to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d908c8a2bb746599a47106b5dcdd6fe 2024-11-11T16:16:39,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/013a170920c24917b7261aae9da29920 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/013a170920c24917b7261aae9da29920 2024-11-11T16:16:39,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/79b1f6dcd1404582a39723f98c0dd6bc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/79b1f6dcd1404582a39723f98c0dd6bc 2024-11-11T16:16:39,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4af36e1f356e468992aea45e71c79fb5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/4af36e1f356e468992aea45e71c79fb5 2024-11-11T16:16:39,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/356a9fa87b9d426a894b33096d6b923d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/356a9fa87b9d426a894b33096d6b923d 2024-11-11T16:16:39,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/633d2a4c87454db08f5eb3a9aece8d5b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/633d2a4c87454db08f5eb3a9aece8d5b 2024-11-11T16:16:39,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a74d4e5f80ca489b909046e9f45833e3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/a74d4e5f80ca489b909046e9f45833e3 2024-11-11T16:16:39,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/3ece60c6f2a543caba2a42b673009e46 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/3ece60c6f2a543caba2a42b673009e46 2024-11-11T16:16:39,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2e90c0ae84804a1b8bedcf2b9973a33f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2e90c0ae84804a1b8bedcf2b9973a33f 2024-11-11T16:16:39,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d8faa4ac950743a4b7da15db6dbcf437 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/d8faa4ac950743a4b7da15db6dbcf437 2024-11-11T16:16:39,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/242967ee1cf545118326f25e2d002747 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/242967ee1cf545118326f25e2d002747 2024-11-11T16:16:39,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9cc902d6968346be92ea86fd329bb1b8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9cc902d6968346be92ea86fd329bb1b8 2024-11-11T16:16:39,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2be3be5d41674ada9f60ecdcffb4f53c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/2be3be5d41674ada9f60ecdcffb4f53c 2024-11-11T16:16:39,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/91d93bf1bd03482a96ac02267d40fe82 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/91d93bf1bd03482a96ac02267d40fe82 2024-11-11T16:16:39,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/0f2cb47bdeeb4bf18f2591730ad82f4d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/0f2cb47bdeeb4bf18f2591730ad82f4d 2024-11-11T16:16:39,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/cee56a599e5c41b48ed77919b26d19df to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/cee56a599e5c41b48ed77919b26d19df 2024-11-11T16:16:39,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/6df08ac1027849e09f3e82b2b23a2b81 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/6df08ac1027849e09f3e82b2b23a2b81 2024-11-11T16:16:39,782 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/recovered.edits/587.seqid, newMaxSeqId=587, maxSeqId=1 2024-11-11T16:16:39,783 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. 2024-11-11T16:16:39,783 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 9e768d9f517a9b5cf0bf527b36f5956b: Waiting for close lock at 1731341798439Running coprocessor pre-close hooks at 1731341798439Disabling compacts and flushes for region at 1731341798439Disabling writes for close at 1731341798439Obtaining lock to block concurrent updates at 1731341798439Preparing flush snapshotting stores in 9e768d9f517a9b5cf0bf527b36f5956b at 1731341798439Finished memstore snapshotting TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731341798440 (+1 ms)Flushing stores of TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b. at 1731341798440Flushing 9e768d9f517a9b5cf0bf527b36f5956b/A: creating writer at 1731341798440Flushing 9e768d9f517a9b5cf0bf527b36f5956b/A: appending metadata at 1731341798444 (+4 ms)Flushing 9e768d9f517a9b5cf0bf527b36f5956b/A: closing flushed file at 1731341798444Flushing 9e768d9f517a9b5cf0bf527b36f5956b/B: creating writer at 1731341798853 (+409 ms)Flushing 9e768d9f517a9b5cf0bf527b36f5956b/B: appending metadata at 1731341798855 (+2 ms)Flushing 9e768d9f517a9b5cf0bf527b36f5956b/B: closing flushed file at 1731341798855Flushing 9e768d9f517a9b5cf0bf527b36f5956b/C: creating writer at 1731341799262 (+407 ms)Flushing 9e768d9f517a9b5cf0bf527b36f5956b/C: appending metadata at 1731341799264 (+2 ms)Flushing 9e768d9f517a9b5cf0bf527b36f5956b/C: closing flushed file at 1731341799264Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4168b5fd: reopening flushed file at 1731341799671 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@447d24eb: reopening flushed file at 1731341799675 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b55f2: reopening flushed file at 1731341799679 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9e768d9f517a9b5cf0bf527b36f5956b in 1244ms, sequenceid=584, compaction requested=true at 1731341799684 (+5 ms)Writing region close event to WAL at 1731341799780 (+96 ms)Running coprocessor post-close hooks at 1731341799783 (+3 ms)Closed at 1731341799783 2024-11-11T16:16:39,785 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:39,785 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=9e768d9f517a9b5cf0bf527b36f5956b, regionState=CLOSED 2024-11-11T16:16:39,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:16:39,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T16:16:39,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e768d9f517a9b5cf0bf527b36f5956b, server=16b413a53992,45121,1731341648557 in 1.5010 sec 2024-11-11T16:16:39,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-11T16:16:39,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e768d9f517a9b5cf0bf527b36f5956b, UNASSIGN in 1.5050 sec 2024-11-11T16:16:39,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T16:16:39,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-11T16:16:39,794 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341799794"}]},"ts":"1731341799794"} 2024-11-11T16:16:39,795 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T16:16:39,795 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T16:16:39,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-11T16:16:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T16:16:40,426 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:40,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T16:16:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:40,428 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T16:16:40,429 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:40,430 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:40,433 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/recovered.edits] 2024-11-11T16:16:40,435 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/44c8d16bda7941c19c8b3db518493762 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/44c8d16bda7941c19c8b3db518493762 2024-11-11T16:16:40,436 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/885df65ef9a14480b50be9bc506d21d9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/885df65ef9a14480b50be9bc506d21d9 2024-11-11T16:16:40,437 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/aade8c93c7f14139aaf412585ed704aa to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/A/aade8c93c7f14139aaf412585ed704aa 2024-11-11T16:16:40,439 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1b7358de43684e17b2c8d4c255c45969 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/1b7358de43684e17b2c8d4c255c45969 2024-11-11T16:16:40,440 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/43bb7cd4ec5e4327a87155c115d937c6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/43bb7cd4ec5e4327a87155c115d937c6 2024-11-11T16:16:40,441 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/749c32859ed645aba15d2ff7aa00849f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/B/749c32859ed645aba15d2ff7aa00849f 2024-11-11T16:16:40,442 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/647c33b47c3c4c078cd5ac945242bce7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/647c33b47c3c4c078cd5ac945242bce7 2024-11-11T16:16:40,443 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d473448367e4a64a36466984e6dade7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9d473448367e4a64a36466984e6dade7 2024-11-11T16:16:40,444 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9e248340c92547d99f4141e0d85ea578 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/C/9e248340c92547d99f4141e0d85ea578 2024-11-11T16:16:40,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/recovered.edits/587.seqid to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b/recovered.edits/587.seqid 2024-11-11T16:16:40,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/9e768d9f517a9b5cf0bf527b36f5956b 2024-11-11T16:16:40,446 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T16:16:40,448 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:40,450 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T16:16:40,451 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T16:16:40,452 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:40,452 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T16:16:40,452 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731341800452"}]},"ts":"9223372036854775807"} 2024-11-11T16:16:40,454 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T16:16:40,454 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9e768d9f517a9b5cf0bf527b36f5956b, NAME => 'TestAcidGuarantees,,1731341772795.9e768d9f517a9b5cf0bf527b36f5956b.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T16:16:40,454 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T16:16:40,454 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731341800454"}]},"ts":"9223372036854775807"} 2024-11-11T16:16:40,455 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T16:16:40,456 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:40,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-11T16:16:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T16:16:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T16:16:40,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:40,546 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 252), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=532 (was 600), ProcessCount=11 (was 11), AvailableMemoryMB=1891 (was 302) - AvailableMemoryMB LEAK? - 2024-11-11T16:16:40,557 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=532, ProcessCount=11, AvailableMemoryMB=1890 2024-11-11T16:16:40,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:16:40,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:16:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:40,560 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T16:16:40,560 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:40,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-11T16:16:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T16:16:40,561 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T16:16:40,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742392_1568 (size=1061) 2024-11-11T16:16:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T16:16:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T16:16:40,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669 2024-11-11T16:16:40,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742393_1569 (size=53) 2024-11-11T16:16:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T16:16:41,373 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:16:41,373 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 70a39dbdbb2217d780a9c88575e9f99d, disabling compactions & flushes 2024-11-11T16:16:41,373 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,373 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,373 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. after waiting 0 ms 2024-11-11T16:16:41,373 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,373 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,373 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 70a39dbdbb2217d780a9c88575e9f99d: Waiting for close lock at 1731341801373Disabling compacts and flushes for region at 1731341801373Disabling writes for close at 1731341801373Writing region close event to WAL at 1731341801373Closed at 1731341801373 2024-11-11T16:16:41,375 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T16:16:41,375 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731341801375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731341801375"}]},"ts":"1731341801375"} 2024-11-11T16:16:41,376 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T16:16:41,377 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T16:16:41,377 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341801377"}]},"ts":"1731341801377"} 2024-11-11T16:16:41,378 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T16:16:41,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, ASSIGN}] 2024-11-11T16:16:41,379 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, ASSIGN 2024-11-11T16:16:41,380 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, ASSIGN; state=OFFLINE, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=false 2024-11-11T16:16:41,530 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:41,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, ASSIGN because future has completed 2024-11-11T16:16:41,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:16:41,686 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:16:41,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:16:41,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,687 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,688 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,689 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:41,689 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70a39dbdbb2217d780a9c88575e9f99d columnFamilyName A 2024-11-11T16:16:41,689 DEBUG [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:41,690 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(327): Store=70a39dbdbb2217d780a9c88575e9f99d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:41,690 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,691 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:41,691 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70a39dbdbb2217d780a9c88575e9f99d columnFamilyName B 2024-11-11T16:16:41,691 DEBUG [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:41,691 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(327): Store=70a39dbdbb2217d780a9c88575e9f99d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:41,691 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,692 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:41,692 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70a39dbdbb2217d780a9c88575e9f99d columnFamilyName C 2024-11-11T16:16:41,692 DEBUG [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:41,692 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(327): Store=70a39dbdbb2217d780a9c88575e9f99d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:41,693 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,693 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,693 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,693 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,694 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,694 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,695 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:16:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T16:16:41,696 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,697 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T16:16:41,705 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 70a39dbdbb2217d780a9c88575e9f99d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70408847, jitterRate=0.04917357861995697}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:16:41,705 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:41,706 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 70a39dbdbb2217d780a9c88575e9f99d: Running coprocessor pre-open hook at 1731341801687Writing region info on filesystem at 1731341801687Initializing all the Stores at 1731341801688 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341801688Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341801688Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341801688Cleaning up temporary data from old regions at 1731341801694 (+6 ms)Running coprocessor post-open hooks at 1731341801705 (+11 ms)Region opened successfully at 1731341801706 (+1 ms) 2024-11-11T16:16:41,709 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., pid=138, masterSystemTime=1731341801684 2024-11-11T16:16:41,711 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,711 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:41,711 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=OPEN, openSeqNum=2, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:41,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:16:41,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T16:16:41,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 in 182 msec 2024-11-11T16:16:41,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-11T16:16:41,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, ASSIGN in 337 msec 2024-11-11T16:16:41,718 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T16:16:41,718 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341801718"}]},"ts":"1731341801718"} 2024-11-11T16:16:41,719 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T16:16:41,720 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T16:16:41,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-11T16:16:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T16:16:42,716 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:42,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bedb280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:42,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:42,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:42,718 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582164a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:42,720 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f944d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:42,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:42,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:42,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:42,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:42,724 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T16:16:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b413a53992,33601,1731341647570 2024-11-11T16:16:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@486c8c2e 2024-11-11T16:16:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T16:16:42,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T16:16:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T16:16:42,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T16:16:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T16:16:42,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742394_1570 (size=1097) 2024-11-11T16:16:43,138 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T16:16:43,138 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T16:16:43,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:16:43,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, REOPEN/MOVE}] 2024-11-11T16:16:43,142 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, REOPEN/MOVE 2024-11-11T16:16:43,143 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, REOPEN/MOVE because future has completed 2024-11-11T16:16:43,144 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:16:43,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:16:43,295 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,295 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:16:43,296 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 70a39dbdbb2217d780a9c88575e9f99d, disabling compactions & flushes 2024-11-11T16:16:43,296 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,296 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,296 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. after waiting 0 ms 2024-11-11T16:16:43,296 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,299 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T16:16:43,300 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,300 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 70a39dbdbb2217d780a9c88575e9f99d: Waiting for close lock at 1731341803296Running coprocessor pre-close hooks at 1731341803296Disabling compacts and flushes for region at 1731341803296Disabling writes for close at 1731341803296Writing region close event to WAL at 1731341803296Running coprocessor post-close hooks at 1731341803300 (+4 ms)Closed at 1731341803300 2024-11-11T16:16:43,300 WARN [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 70a39dbdbb2217d780a9c88575e9f99d to self. 2024-11-11T16:16:43,301 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,302 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=CLOSED 2024-11-11T16:16:43,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:16:43,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T16:16:43,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 in 160 msec 2024-11-11T16:16:43,306 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, REOPEN/MOVE; state=CLOSED, location=16b413a53992,45121,1731341648557; forceNewPlan=false, retain=true 2024-11-11T16:16:43,457 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=OPENING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, REOPEN/MOVE because future has completed 2024-11-11T16:16:43,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:16:43,613 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,613 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} 2024-11-11T16:16:43,613 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,613 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T16:16:43,614 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,614 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,615 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,615 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:43,616 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70a39dbdbb2217d780a9c88575e9f99d columnFamilyName A 2024-11-11T16:16:43,617 DEBUG [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:43,617 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(327): Store=70a39dbdbb2217d780a9c88575e9f99d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:43,617 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,618 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:43,618 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70a39dbdbb2217d780a9c88575e9f99d columnFamilyName B 2024-11-11T16:16:43,618 DEBUG [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:43,618 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(327): Store=70a39dbdbb2217d780a9c88575e9f99d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:43,618 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,619 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T16:16:43,619 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70a39dbdbb2217d780a9c88575e9f99d columnFamilyName C 2024-11-11T16:16:43,619 DEBUG [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:43,619 INFO [StoreOpener-70a39dbdbb2217d780a9c88575e9f99d-1 {}] regionserver.HStore(327): Store=70a39dbdbb2217d780a9c88575e9f99d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T16:16:43,619 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,619 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,620 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,620 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,621 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,621 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,621 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T16:16:43,622 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,622 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 70a39dbdbb2217d780a9c88575e9f99d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62055273, jitterRate=-0.07530437409877777}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T16:16:43,623 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,623 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 70a39dbdbb2217d780a9c88575e9f99d: Running coprocessor pre-open hook at 1731341803614Writing region info on filesystem at 1731341803614Initializing all the Stores at 1731341803614Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341803614Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341803615 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731341803615Cleaning up temporary data from old regions at 1731341803621 (+6 ms)Running coprocessor post-open hooks at 1731341803623 (+2 ms)Region opened successfully at 1731341803623 2024-11-11T16:16:43,624 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., pid=143, masterSystemTime=1731341803610 2024-11-11T16:16:43,626 DEBUG [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,626 INFO [RS_OPEN_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,626 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=OPEN, openSeqNum=5, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:16:43,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-11T16:16:43,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 in 169 msec 2024-11-11T16:16:43,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-11T16:16:43,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, REOPEN/MOVE in 487 msec 2024-11-11T16:16:43,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T16:16:43,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-11T16:16:43,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-11T16:16:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T16:16:43,635 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:43,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fff8313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,636 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2326d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,638 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cdde89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e671b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,639 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f3381e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,641 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74455e12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca2e1a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,642 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2760b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5f38cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5a87ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19cdd2a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,646 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af32bfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203b44cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,649 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75525a4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,650 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32053010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69473cad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,651 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e02eb47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,653 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cfac91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d69eb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251c8d7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2dea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5136b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,657 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592ef3a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,658 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b36d041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba99a34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,660 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27baded4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,661 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a957e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cad871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b413a53992,33601,-1 for getting cluster id 2024-11-11T16:16:43,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T16:16:43,662 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bef5b00d-7213-459c-b3e4-26a5a980fa1f' 2024-11-11T16:16:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T16:16:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bef5b00d-7213-459c-b3e4-26a5a980fa1f" 2024-11-11T16:16:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89a09b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b413a53992,33601,-1] 2024-11-11T16:16:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T16:16:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:16:43,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T16:16:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@497c5f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T16:16:43,664 DEBUG [Thread-2525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,664 DEBUG [Thread-2544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,664 DEBUG [Thread-2527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,664 DEBUG [Thread-2531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,665 DEBUG [Thread-2540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,665 DEBUG [Thread-2536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,665 DEBUG [Thread-2529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,666 DEBUG [Thread-2542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,667 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,667 DEBUG [Thread-2533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,668 DEBUG [Thread-2538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T16:16:43,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b413a53992,45121,1731341648557, seqNum=-1] 2024-11-11T16:16:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T16:16:43,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T16:16:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-11T16:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T16:16:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,675 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5] 2024-11-11T16:16:43,676 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:43,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T16:16:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:43,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731341803701, max timestamp=1731341803712Num uniques 859; 2024-11-11T16:16:43,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:43,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731341803701, max timestamp=1731341803712Num uniques 859; 2024-11-11T16:16:43,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:43,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341803701, max timestamp=1731341803712Num uniques 859; 2024-11-11T16:16:43,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T16:16:43,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b586b536a9fc4a4ebaba31841bab44e8_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341803681/Put/seqid=0 2024-11-11T16:16:43,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742395_1571 (size=12154) 2024-11-11T16:16:43,739 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341863733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341863735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341863736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341863737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b586b536a9fc4a4ebaba31841bab44e8_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b586b536a9fc4a4ebaba31841bab44e8_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341863739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,746 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/87497efbf56b483bb6290eaedbe0fa5c, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:43,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/87497efbf56b483bb6290eaedbe0fa5c is 175, key is test_row_0/A:col10/1731341803681/Put/seqid=0 2024-11-11T16:16:43,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742396_1572 (size=30955) 2024-11-11T16:16:43,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/87497efbf56b483bb6290eaedbe0fa5c 2024-11-11T16:16:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T16:16:43,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/188dbd102b494c77b5676aad16797434 is 50, key is test_row_0/B:col10/1731341803681/Put/seqid=0 2024-11-11T16:16:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742397_1573 (size=12001) 2024-11-11T16:16:43,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T16:16:43,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:43,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:43,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:43,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341863846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341863846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341863847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341863847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341863856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:43,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T16:16:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T16:16:43,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:43,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:43,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:43,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:44,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341864056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341864056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341864056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341864056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341864065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T16:16:44,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:44,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:44,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:44,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:44,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:44,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/188dbd102b494c77b5676aad16797434 2024-11-11T16:16:44,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/ca6b5d7873134992883a78b5dd3e7d78 is 50, key is test_row_0/C:col10/1731341803681/Put/seqid=0 2024-11-11T16:16:44,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742398_1574 (size=12001) 2024-11-11T16:16:44,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/ca6b5d7873134992883a78b5dd3e7d78 2024-11-11T16:16:44,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/87497efbf56b483bb6290eaedbe0fa5c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/87497efbf56b483bb6290eaedbe0fa5c 2024-11-11T16:16:44,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/87497efbf56b483bb6290eaedbe0fa5c, entries=150, sequenceid=16, filesize=30.2 K 2024-11-11T16:16:44,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/188dbd102b494c77b5676aad16797434 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/188dbd102b494c77b5676aad16797434 2024-11-11T16:16:44,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/188dbd102b494c77b5676aad16797434, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T16:16:44,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/ca6b5d7873134992883a78b5dd3e7d78 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/ca6b5d7873134992883a78b5dd3e7d78 2024-11-11T16:16:44,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/ca6b5d7873134992883a78b5dd3e7d78, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T16:16:44,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.13 KB/120966 for 70a39dbdbb2217d780a9c88575e9f99d in 582ms, sequenceid=16, compaction requested=false 2024-11-11T16:16:44,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:44,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T16:16:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:44,292 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=118.13 KB heapSize=334.46 KB 2024-11-11T16:16:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T16:16:44,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ed76d58a69604edc81fd1a8948846116_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341803733/Put/seqid=0 2024-11-11T16:16:44,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742399_1575 (size=12154) 2024-11-11T16:16:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:44,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341864376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341864377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341864382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341864383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341864383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341864485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341864495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341864496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341864496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341864496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341864696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341864706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341864706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341864706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341864706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:44,719 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ed76d58a69604edc81fd1a8948846116_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ed76d58a69604edc81fd1a8948846116_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/19e2f55a126f40398dd198222bf25f22, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:44,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/19e2f55a126f40398dd198222bf25f22 is 175, key is test_row_0/A:col10/1731341803733/Put/seqid=0 2024-11-11T16:16:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742400_1576 (size=30955) 2024-11-11T16:16:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T16:16:45,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341865005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341865016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341865016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341865016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341865016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,129 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/19e2f55a126f40398dd198222bf25f22 2024-11-11T16:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/47926eb9e6c44685916fb0642ba57044 is 50, key is test_row_0/B:col10/1731341803733/Put/seqid=0 2024-11-11T16:16:45,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742401_1577 (size=12001) 2024-11-11T16:16:45,312 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T16:16:45,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341865515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341865526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341865526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341865526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341865526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:45,549 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/47926eb9e6c44685916fb0642ba57044 2024-11-11T16:16:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/0c1be218e6324fb1a9862cfd5d668820 is 50, key is test_row_0/C:col10/1731341803733/Put/seqid=0 2024-11-11T16:16:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742402_1578 (size=12001) 2024-11-11T16:16:45,588 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/0c1be218e6324fb1a9862cfd5d668820 2024-11-11T16:16:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/19e2f55a126f40398dd198222bf25f22 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/19e2f55a126f40398dd198222bf25f22 2024-11-11T16:16:45,598 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/19e2f55a126f40398dd198222bf25f22, entries=150, sequenceid=50, filesize=30.2 K 2024-11-11T16:16:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/47926eb9e6c44685916fb0642ba57044 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/47926eb9e6c44685916fb0642ba57044 2024-11-11T16:16:45,605 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/47926eb9e6c44685916fb0642ba57044, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T16:16:45,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/0c1be218e6324fb1a9862cfd5d668820 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0c1be218e6324fb1a9862cfd5d668820 2024-11-11T16:16:45,610 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0c1be218e6324fb1a9862cfd5d668820, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T16:16:45,611 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~118.13 KB/120966, heapSize ~334.41 KB/342440, currentSize=67.09 KB/68700 for 70a39dbdbb2217d780a9c88575e9f99d in 1319ms, sequenceid=50, compaction requested=false 2024-11-11T16:16:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-11T16:16:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-11T16:16:45,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-11T16:16:45,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9370 sec 2024-11-11T16:16:45,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.9510 sec 2024-11-11T16:16:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T16:16:45,816 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:45,816 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T16:16:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T16:16:45,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:45,820 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:45,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T16:16:45,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T16:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:45,973 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e66a281cba04d7cb459445b53aaa21b_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341804382/Put/seqid=0 2024-11-11T16:16:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742403_1579 (size=12154) 2024-11-11T16:16:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T16:16:46,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T16:16:46,454 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e66a281cba04d7cb459445b53aaa21b_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e66a281cba04d7cb459445b53aaa21b_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/c84b41d5baeb427fb9c084666b3090f1, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/c84b41d5baeb427fb9c084666b3090f1 is 175, key is test_row_0/A:col10/1731341804382/Put/seqid=0 2024-11-11T16:16:46,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742404_1580 (size=30955) 2024-11-11T16:16:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:46,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341866562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341866562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341866564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341866570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341866571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341866676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341866676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341866677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341866686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341866686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,886 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/c84b41d5baeb427fb9c084666b3090f1 2024-11-11T16:16:46,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341866885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341866886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341866886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/bd742529544d4f5fadb80d9c4ab486bc is 50, key is test_row_0/B:col10/1731341804382/Put/seqid=0 2024-11-11T16:16:46,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341866898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341866898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:46,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742405_1581 (size=12001) 2024-11-11T16:16:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T16:16:47,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341867197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341867205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341867206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341867206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341867216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,329 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/bd742529544d4f5fadb80d9c4ab486bc 2024-11-11T16:16:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/7ed3a8f7961d4b16a861bcad2e6ec0fb is 50, key is test_row_0/C:col10/1731341804382/Put/seqid=0 2024-11-11T16:16:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742406_1582 (size=12001) 2024-11-11T16:16:47,343 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/7ed3a8f7961d4b16a861bcad2e6ec0fb 2024-11-11T16:16:47,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/c84b41d5baeb427fb9c084666b3090f1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/c84b41d5baeb427fb9c084666b3090f1 2024-11-11T16:16:47,350 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/c84b41d5baeb427fb9c084666b3090f1, entries=150, sequenceid=63, filesize=30.2 K 2024-11-11T16:16:47,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/bd742529544d4f5fadb80d9c4ab486bc as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/bd742529544d4f5fadb80d9c4ab486bc 2024-11-11T16:16:47,355 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/bd742529544d4f5fadb80d9c4ab486bc, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T16:16:47,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/7ed3a8f7961d4b16a861bcad2e6ec0fb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7ed3a8f7961d4b16a861bcad2e6ec0fb 2024-11-11T16:16:47,360 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7ed3a8f7961d4b16a861bcad2e6ec0fb, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T16:16:47,361 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 70a39dbdbb2217d780a9c88575e9f99d in 1388ms, sequenceid=63, compaction requested=true 2024-11-11T16:16:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:47,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:47,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T16:16:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T16:16:47,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T16:16:47,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-11T16:16:47,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.5540 sec 2024-11-11T16:16:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:47,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T16:16:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:47,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111575dcb4275b74fa3ac7ffa3affb2035a_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341806568/Put/seqid=0 2024-11-11T16:16:47,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742407_1583 (size=12154) 2024-11-11T16:16:47,727 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:47,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111575dcb4275b74fa3ac7ffa3affb2035a_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111575dcb4275b74fa3ac7ffa3affb2035a_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341867734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341867736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341867737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341867737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341867743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/315f18f681ca42b2b92a56f9d9b89b79, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:47,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/315f18f681ca42b2b92a56f9d9b89b79 is 175, key is test_row_0/A:col10/1731341806568/Put/seqid=0 2024-11-11T16:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742408_1584 (size=30955) 2024-11-11T16:16:47,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/315f18f681ca42b2b92a56f9d9b89b79 2024-11-11T16:16:47,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/92d4baf730c240f5af7789e9dacd5f9a is 50, key is test_row_0/B:col10/1731341806568/Put/seqid=0 2024-11-11T16:16:47,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742409_1585 (size=12001) 2024-11-11T16:16:47,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341867846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341867856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341867856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341867856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:47,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341867856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:47,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T16:16:47,966 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:47,967 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T16:16:47,969 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T16:16:47,970 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:47,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:48,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341868057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341868067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341868067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341868069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T16:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341868081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:48,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:48,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/92d4baf730c240f5af7789e9dacd5f9a 2024-11-11T16:16:48,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T16:16:48,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/2cbe4077b6c5432dbe7c23d38312485c is 50, key is test_row_0/C:col10/1731341806568/Put/seqid=0 2024-11-11T16:16:48,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742410_1586 (size=12001) 2024-11-11T16:16:48,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/2cbe4077b6c5432dbe7c23d38312485c 2024-11-11T16:16:48,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/315f18f681ca42b2b92a56f9d9b89b79 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/315f18f681ca42b2b92a56f9d9b89b79 2024-11-11T16:16:48,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/315f18f681ca42b2b92a56f9d9b89b79, entries=150, sequenceid=84, filesize=30.2 K 2024-11-11T16:16:48,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/92d4baf730c240f5af7789e9dacd5f9a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/92d4baf730c240f5af7789e9dacd5f9a 2024-11-11T16:16:48,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/92d4baf730c240f5af7789e9dacd5f9a, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T16:16:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/2cbe4077b6c5432dbe7c23d38312485c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/2cbe4077b6c5432dbe7c23d38312485c 2024-11-11T16:16:48,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/2cbe4077b6c5432dbe7c23d38312485c, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T16:16:48,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 70a39dbdbb2217d780a9c88575e9f99d in 666ms, sequenceid=84, compaction requested=true 2024-11-11T16:16:48,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:48,378 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:16:48,381 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:48,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:16:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:48,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:48,393 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:48,393 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:48,393 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/B is initiating minor compaction (all files) 2024-11-11T16:16:48,393 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/A is initiating minor compaction (all files) 2024-11-11T16:16:48,393 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/A in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,394 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/B in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,394 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/188dbd102b494c77b5676aad16797434, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/47926eb9e6c44685916fb0642ba57044, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/bd742529544d4f5fadb80d9c4ab486bc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/92d4baf730c240f5af7789e9dacd5f9a] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=46.9 K 2024-11-11T16:16:48,394 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/87497efbf56b483bb6290eaedbe0fa5c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/19e2f55a126f40398dd198222bf25f22, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/c84b41d5baeb427fb9c084666b3090f1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/315f18f681ca42b2b92a56f9d9b89b79] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=120.9 K 2024-11-11T16:16:48,394 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,394 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/87497efbf56b483bb6290eaedbe0fa5c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/19e2f55a126f40398dd198222bf25f22, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/c84b41d5baeb427fb9c084666b3090f1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/315f18f681ca42b2b92a56f9d9b89b79] 2024-11-11T16:16:48,394 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188dbd102b494c77b5676aad16797434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341803679 2024-11-11T16:16:48,394 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 87497efbf56b483bb6290eaedbe0fa5c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341803679 2024-11-11T16:16:48,395 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47926eb9e6c44685916fb0642ba57044, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731341803733 2024-11-11T16:16:48,395 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e2f55a126f40398dd198222bf25f22, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731341803733 2024-11-11T16:16:48,406 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd742529544d4f5fadb80d9c4ab486bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731341804375 2024-11-11T16:16:48,407 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c84b41d5baeb427fb9c084666b3090f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731341804375 2024-11-11T16:16:48,407 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d4baf730c240f5af7789e9dacd5f9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341806568 2024-11-11T16:16:48,407 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 315f18f681ca42b2b92a56f9d9b89b79, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341806568 2024-11-11T16:16:48,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116c7fb3cae7ec47dfb50f539898b59068_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341808376/Put/seqid=0 2024-11-11T16:16:48,453 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:48,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341868438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341868444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,460 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#B#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:48,461 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/ec8fcac55a6049ccadcb3333924b0386 is 50, key is test_row_0/B:col10/1731341806568/Put/seqid=0 2024-11-11T16:16:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341868452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341868454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341868454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,475 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117b3103bf8e174dfaaa0e843f5c6ab97e_70a39dbdbb2217d780a9c88575e9f99d store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:48,477 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117b3103bf8e174dfaaa0e843f5c6ab97e_70a39dbdbb2217d780a9c88575e9f99d, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:48,477 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117b3103bf8e174dfaaa0e843f5c6ab97e_70a39dbdbb2217d780a9c88575e9f99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:48,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742411_1587 (size=12154) 2024-11-11T16:16:48,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742412_1588 (size=12139) 2024-11-11T16:16:48,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742413_1589 (size=4469) 2024-11-11T16:16:48,570 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#A#compaction#502 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:48,570 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/646783b08fd24b6083539835bb0c4421 is 175, key is test_row_0/A:col10/1731341806568/Put/seqid=0 2024-11-11T16:16:48,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742414_1590 (size=31093) 2024-11-11T16:16:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341868566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341868566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,584 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/646783b08fd24b6083539835bb0c4421 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/646783b08fd24b6083539835bb0c4421 2024-11-11T16:16:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341868577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341868577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,594 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/A of 70a39dbdbb2217d780a9c88575e9f99d into 646783b08fd24b6083539835bb0c4421(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:48,594 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:48,594 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/A, priority=12, startTime=1731341808378; duration=0sec 2024-11-11T16:16:48,594 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:48,594 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:A 2024-11-11T16:16:48,594 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:48,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:48,595 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,595 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/C is initiating minor compaction (all files) 2024-11-11T16:16:48,595 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/C in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,595 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/ca6b5d7873134992883a78b5dd3e7d78, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0c1be218e6324fb1a9862cfd5d668820, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7ed3a8f7961d4b16a861bcad2e6ec0fb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/2cbe4077b6c5432dbe7c23d38312485c] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=46.9 K 2024-11-11T16:16:48,596 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ca6b5d7873134992883a78b5dd3e7d78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731341803679 2024-11-11T16:16:48,596 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1be218e6324fb1a9862cfd5d668820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731341803733 2024-11-11T16:16:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,596 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed3a8f7961d4b16a861bcad2e6ec0fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731341804375 2024-11-11T16:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T16:16:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341868582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,598 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cbe4077b6c5432dbe7c23d38312485c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341806568 2024-11-11T16:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,647 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#C#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:48,648 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/b78cf19e32384423923b1b6aec7fa3a6 is 50, key is test_row_0/C:col10/1731341806568/Put/seqid=0 2024-11-11T16:16:48,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742415_1591 (size=12139) 2024-11-11T16:16:48,697 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/b78cf19e32384423923b1b6aec7fa3a6 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/b78cf19e32384423923b1b6aec7fa3a6 2024-11-11T16:16:48,701 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/C of 70a39dbdbb2217d780a9c88575e9f99d into b78cf19e32384423923b1b6aec7fa3a6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:48,701 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:48,701 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/C, priority=12, startTime=1731341808378; duration=0sec 2024-11-11T16:16:48,701 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:48,701 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:C 2024-11-11T16:16:48,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341868786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341868786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341868796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341868796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341868806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:48,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:48,918 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116c7fb3cae7ec47dfb50f539898b59068_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116c7fb3cae7ec47dfb50f539898b59068_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:48,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:48,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:48,925 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2706fd1b03aa466b81f2f175de312f98, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:48,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2706fd1b03aa466b81f2f175de312f98 is 175, key is test_row_0/A:col10/1731341808376/Put/seqid=0 2024-11-11T16:16:48,952 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/ec8fcac55a6049ccadcb3333924b0386 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/ec8fcac55a6049ccadcb3333924b0386 2024-11-11T16:16:48,956 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/B of 70a39dbdbb2217d780a9c88575e9f99d into ec8fcac55a6049ccadcb3333924b0386(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:48,956 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:48,956 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/B, priority=12, startTime=1731341808378; duration=0sec 2024-11-11T16:16:48,956 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:48,956 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:B 2024-11-11T16:16:48,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742416_1592 (size=30955) 2024-11-11T16:16:48,967 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2706fd1b03aa466b81f2f175de312f98 2024-11-11T16:16:48,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/50be2990dd864b778e158a1095a35a70 is 50, key is test_row_0/B:col10/1731341808376/Put/seqid=0 2024-11-11T16:16:48,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742417_1593 (size=12001) 2024-11-11T16:16:49,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:49,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341869096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341869096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T16:16:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341869105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341869106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341869116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/50be2990dd864b778e158a1095a35a70 2024-11-11T16:16:49,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/05c8d49996954ce9b3a70ae3326a16ba is 50, key is test_row_0/C:col10/1731341808376/Put/seqid=0 2024-11-11T16:16:49,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742418_1594 (size=12001) 2024-11-11T16:16:49,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341869609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341869609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341869615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341869615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341869629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:49,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/05c8d49996954ce9b3a70ae3326a16ba 2024-11-11T16:16:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2706fd1b03aa466b81f2f175de312f98 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2706fd1b03aa466b81f2f175de312f98 2024-11-11T16:16:49,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2706fd1b03aa466b81f2f175de312f98, entries=150, sequenceid=97, filesize=30.2 K 2024-11-11T16:16:49,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/50be2990dd864b778e158a1095a35a70 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/50be2990dd864b778e158a1095a35a70 2024-11-11T16:16:49,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:49,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:49,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/50be2990dd864b778e158a1095a35a70, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T16:16:49,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/05c8d49996954ce9b3a70ae3326a16ba as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/05c8d49996954ce9b3a70ae3326a16ba 2024-11-11T16:16:49,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/05c8d49996954ce9b3a70ae3326a16ba, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T16:16:49,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 70a39dbdbb2217d780a9c88575e9f99d in 1476ms, sequenceid=97, compaction requested=false 2024-11-11T16:16:49,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:50,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T16:16:50,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:50,005 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:16:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:50,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118397ccb56cda4dae8d0940c7ce36b3d1_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341808450/Put/seqid=0 2024-11-11T16:16:50,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742419_1595 (size=12154) 2024-11-11T16:16:50,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:50,064 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118397ccb56cda4dae8d0940c7ce36b3d1_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118397ccb56cda4dae8d0940c7ce36b3d1_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/5b0c6e718db6455cb1e3800b679bab45, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/5b0c6e718db6455cb1e3800b679bab45 is 175, key is test_row_0/A:col10/1731341808450/Put/seqid=0 2024-11-11T16:16:50,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742420_1596 (size=30955) 2024-11-11T16:16:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T16:16:50,484 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/5b0c6e718db6455cb1e3800b679bab45 2024-11-11T16:16:50,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/6e12905d826f40b0861ae194357f00bb is 50, key is test_row_0/B:col10/1731341808450/Put/seqid=0 2024-11-11T16:16:50,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742421_1597 (size=12001) 2024-11-11T16:16:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:50,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341870649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341870653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341870658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341870665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341870673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341870772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341870776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341870776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341870789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341870789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:50,909 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/6e12905d826f40b0861ae194357f00bb 2024-11-11T16:16:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/af4d7df3e6cc4a0caeebfe638957247b is 50, key is test_row_0/C:col10/1731341808450/Put/seqid=0 2024-11-11T16:16:50,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742422_1598 (size=12001) 2024-11-11T16:16:50,941 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/af4d7df3e6cc4a0caeebfe638957247b 2024-11-11T16:16:50,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/5b0c6e718db6455cb1e3800b679bab45 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/5b0c6e718db6455cb1e3800b679bab45 2024-11-11T16:16:50,957 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/5b0c6e718db6455cb1e3800b679bab45, entries=150, sequenceid=119, filesize=30.2 K 2024-11-11T16:16:50,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/6e12905d826f40b0861ae194357f00bb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/6e12905d826f40b0861ae194357f00bb 2024-11-11T16:16:50,961 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/6e12905d826f40b0861ae194357f00bb, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T16:16:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/af4d7df3e6cc4a0caeebfe638957247b as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af4d7df3e6cc4a0caeebfe638957247b 2024-11-11T16:16:50,966 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af4d7df3e6cc4a0caeebfe638957247b, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T16:16:50,967 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 70a39dbdbb2217d780a9c88575e9f99d in 962ms, sequenceid=119, compaction requested=true 2024-11-11T16:16:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T16:16:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T16:16:50,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T16:16:50,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0000 sec 2024-11-11T16:16:50,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.0060 sec 2024-11-11T16:16:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:50,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T16:16:50,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:50,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:50,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:50,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:50,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:50,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:51,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c1d7583525914f25bff0c7a9e14fbbf1_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341810657/Put/seqid=0 2024-11-11T16:16:51,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742423_1599 (size=12304) 2024-11-11T16:16:51,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341871014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341871019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c1d7583525914f25bff0c7a9e14fbbf1_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c1d7583525914f25bff0c7a9e14fbbf1_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:51,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/11104a78e92e480e89e41c4aff1db276, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:51,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/11104a78e92e480e89e41c4aff1db276 is 175, key is test_row_0/A:col10/1731341810657/Put/seqid=0 2024-11-11T16:16:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341871022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341871025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341871025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742424_1600 (size=31105) 2024-11-11T16:16:51,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341871136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341871137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341871137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341871138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341871138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341871346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341871346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341871356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341871356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341871356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,445 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/11104a78e92e480e89e41c4aff1db276 2024-11-11T16:16:51,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/7bd4b3e11bf54cb89aa22039abdbc7c4 is 50, key is test_row_0/B:col10/1731341810657/Put/seqid=0 2024-11-11T16:16:51,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742425_1601 (size=12151) 2024-11-11T16:16:51,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/7bd4b3e11bf54cb89aa22039abdbc7c4 2024-11-11T16:16:51,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/9a302b0dba1741b2acd5f90088585744 is 50, key is test_row_0/C:col10/1731341810657/Put/seqid=0 2024-11-11T16:16:51,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742426_1602 (size=12151) 2024-11-11T16:16:51,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341871656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341871656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341871665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341871666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341871666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:51,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/9a302b0dba1741b2acd5f90088585744 2024-11-11T16:16:51,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/11104a78e92e480e89e41c4aff1db276 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/11104a78e92e480e89e41c4aff1db276 2024-11-11T16:16:51,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/11104a78e92e480e89e41c4aff1db276, entries=150, sequenceid=134, filesize=30.4 K 2024-11-11T16:16:51,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/7bd4b3e11bf54cb89aa22039abdbc7c4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7bd4b3e11bf54cb89aa22039abdbc7c4 2024-11-11T16:16:52,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7bd4b3e11bf54cb89aa22039abdbc7c4, entries=150, sequenceid=134, filesize=11.9 K 2024-11-11T16:16:52,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/9a302b0dba1741b2acd5f90088585744 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/9a302b0dba1741b2acd5f90088585744 2024-11-11T16:16:52,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/9a302b0dba1741b2acd5f90088585744, entries=150, sequenceid=134, filesize=11.9 K 2024-11-11T16:16:52,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 70a39dbdbb2217d780a9c88575e9f99d in 1018ms, sequenceid=134, compaction requested=true 2024-11-11T16:16:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:52,009 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:52,009 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:52,010 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:52,010 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124108 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:52,010 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/A is initiating minor compaction (all files) 2024-11-11T16:16:52,010 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/B is initiating minor compaction (all files) 2024-11-11T16:16:52,010 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/A in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,010 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/B in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,010 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/646783b08fd24b6083539835bb0c4421, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2706fd1b03aa466b81f2f175de312f98, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/5b0c6e718db6455cb1e3800b679bab45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/11104a78e92e480e89e41c4aff1db276] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=121.2 K 2024-11-11T16:16:52,010 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/ec8fcac55a6049ccadcb3333924b0386, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/50be2990dd864b778e158a1095a35a70, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/6e12905d826f40b0861ae194357f00bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7bd4b3e11bf54cb89aa22039abdbc7c4] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=47.2 K 2024-11-11T16:16:52,010 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,010 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/646783b08fd24b6083539835bb0c4421, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2706fd1b03aa466b81f2f175de312f98, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/5b0c6e718db6455cb1e3800b679bab45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/11104a78e92e480e89e41c4aff1db276] 2024-11-11T16:16:52,011 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting ec8fcac55a6049ccadcb3333924b0386, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341806568 2024-11-11T16:16:52,011 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 646783b08fd24b6083539835bb0c4421, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341806568 2024-11-11T16:16:52,012 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 50be2990dd864b778e158a1095a35a70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731341807735 2024-11-11T16:16:52,012 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2706fd1b03aa466b81f2f175de312f98, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731341807735 2024-11-11T16:16:52,012 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e12905d826f40b0861ae194357f00bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731341808429 2024-11-11T16:16:52,012 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0c6e718db6455cb1e3800b679bab45, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731341808429 2024-11-11T16:16:52,013 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd4b3e11bf54cb89aa22039abdbc7c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341810655 2024-11-11T16:16:52,013 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11104a78e92e480e89e41c4aff1db276, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341810655 2024-11-11T16:16:52,021 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:52,029 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#B#compaction#514 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:52,029 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/708d53b81895491e987f4276eb6abdde is 50, key is test_row_0/B:col10/1731341810657/Put/seqid=0 2024-11-11T16:16:52,033 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e9478bf3248d468aaf58359e44e4f020_70a39dbdbb2217d780a9c88575e9f99d store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:52,036 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e9478bf3248d468aaf58359e44e4f020_70a39dbdbb2217d780a9c88575e9f99d, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:52,036 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e9478bf3248d468aaf58359e44e4f020_70a39dbdbb2217d780a9c88575e9f99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:52,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742427_1603 (size=12425) 2024-11-11T16:16:52,063 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/708d53b81895491e987f4276eb6abdde as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/708d53b81895491e987f4276eb6abdde 2024-11-11T16:16:52,069 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/B of 70a39dbdbb2217d780a9c88575e9f99d into 708d53b81895491e987f4276eb6abdde(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:52,069 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:52,069 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/B, priority=12, startTime=1731341812009; duration=0sec 2024-11-11T16:16:52,069 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:52,069 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:B 2024-11-11T16:16:52,070 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:52,072 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:52,072 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/C is initiating minor compaction (all files) 2024-11-11T16:16:52,072 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/C in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,072 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/b78cf19e32384423923b1b6aec7fa3a6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/05c8d49996954ce9b3a70ae3326a16ba, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af4d7df3e6cc4a0caeebfe638957247b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/9a302b0dba1741b2acd5f90088585744] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=47.2 K 2024-11-11T16:16:52,074 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b78cf19e32384423923b1b6aec7fa3a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731341806568 2024-11-11T16:16:52,075 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 05c8d49996954ce9b3a70ae3326a16ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731341807735 2024-11-11T16:16:52,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742428_1604 (size=4469) 2024-11-11T16:16:52,075 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting af4d7df3e6cc4a0caeebfe638957247b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731341808429 2024-11-11T16:16:52,075 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a302b0dba1741b2acd5f90088585744, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341810655 2024-11-11T16:16:52,078 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#A#compaction#513 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:52,078 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/d6848ad377f34ad3bb39855c93119f91 is 175, key is test_row_0/A:col10/1731341810657/Put/seqid=0 2024-11-11T16:16:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742429_1605 (size=31379) 2024-11-11T16:16:52,087 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#C#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:52,088 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/a1614495616b406f8ffbeb8986d198b8 is 50, key is test_row_0/C:col10/1731341810657/Put/seqid=0 2024-11-11T16:16:52,102 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/d6848ad377f34ad3bb39855c93119f91 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/d6848ad377f34ad3bb39855c93119f91 2024-11-11T16:16:52,108 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/A of 70a39dbdbb2217d780a9c88575e9f99d into d6848ad377f34ad3bb39855c93119f91(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:52,108 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:52,108 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/A, priority=12, startTime=1731341812009; duration=0sec 2024-11-11T16:16:52,108 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:52,108 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:A 2024-11-11T16:16:52,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742430_1606 (size=12425) 2024-11-11T16:16:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T16:16:52,126 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:52,126 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T16:16:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T16:16:52,130 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:52,131 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:52,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:52,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:16:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:52,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110d5fd29aa29c434f9ece5bd0d4f64a55_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341811018/Put/seqid=0 2024-11-11T16:16:52,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742431_1607 (size=14794) 2024-11-11T16:16:52,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341872185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341872186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341872188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341872189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341872190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T16:16:52,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341872297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341872297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341872306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341872306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T16:16:52,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:52,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:52,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341872505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341872507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341872516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,519 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/a1614495616b406f8ffbeb8986d198b8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/a1614495616b406f8ffbeb8986d198b8 2024-11-11T16:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341872516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,524 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/C of 70a39dbdbb2217d780a9c88575e9f99d into a1614495616b406f8ffbeb8986d198b8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:52,524 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:52,524 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/C, priority=12, startTime=1731341812009; duration=0sec 2024-11-11T16:16:52,524 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:52,524 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:C 2024-11-11T16:16:52,591 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:52,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,603 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110d5fd29aa29c434f9ece5bd0d4f64a55_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110d5fd29aa29c434f9ece5bd0d4f64a55_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/e558016fb16f4835825da9f86d4b2aac, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:52,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/e558016fb16f4835825da9f86d4b2aac is 175, key is test_row_0/A:col10/1731341811018/Put/seqid=0 2024-11-11T16:16:52,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742432_1608 (size=39749) 2024-11-11T16:16:52,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T16:16:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341872816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341872816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341872825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341872826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:52,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:52,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:52,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,027 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/e558016fb16f4835825da9f86d4b2aac 2024-11-11T16:16:53,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/4008049363c84e9d81f9dfe8b761c50e is 50, key is test_row_0/B:col10/1731341811018/Put/seqid=0 2024-11-11T16:16:53,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742433_1609 (size=12151) 2024-11-11T16:16:53,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/4008049363c84e9d81f9dfe8b761c50e 2024-11-11T16:16:53,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/6c109f1be89645b0bbe25ec6a69e38f8 is 50, key is test_row_0/C:col10/1731341811018/Put/seqid=0 2024-11-11T16:16:53,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742434_1610 (size=12151) 2024-11-11T16:16:53,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341873206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:53,213 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 70a39dbdbb2217d780a9c88575e9f99d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:53,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T16:16:53,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341873327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341873326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341873336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341873336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:53,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:53,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/6c109f1be89645b0bbe25ec6a69e38f8 2024-11-11T16:16:53,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/e558016fb16f4835825da9f86d4b2aac as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/e558016fb16f4835825da9f86d4b2aac 2024-11-11T16:16:53,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/e558016fb16f4835825da9f86d4b2aac, entries=200, sequenceid=155, filesize=38.8 K 2024-11-11T16:16:53,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/4008049363c84e9d81f9dfe8b761c50e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4008049363c84e9d81f9dfe8b761c50e 2024-11-11T16:16:53,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4008049363c84e9d81f9dfe8b761c50e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T16:16:53,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/6c109f1be89645b0bbe25ec6a69e38f8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6c109f1be89645b0bbe25ec6a69e38f8 2024-11-11T16:16:53,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6c109f1be89645b0bbe25ec6a69e38f8, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T16:16:53,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 70a39dbdbb2217d780a9c88575e9f99d in 1354ms, sequenceid=155, compaction requested=false 2024-11-11T16:16:53,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:53,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T16:16:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,538 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T16:16:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:53,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111193f07958c3e54d278fb980b6c2924911_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341812189/Put/seqid=0 2024-11-11T16:16:53,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742435_1611 (size=12304) 2024-11-11T16:16:53,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,573 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111193f07958c3e54d278fb980b6c2924911_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193f07958c3e54d278fb980b6c2924911_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:53,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a74f3041bd6c415cbcd328eccd19fc45, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a74f3041bd6c415cbcd328eccd19fc45 is 175, key is test_row_0/A:col10/1731341812189/Put/seqid=0 2024-11-11T16:16:53,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742436_1612 (size=31105) 2024-11-11T16:16:53,587 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a74f3041bd6c415cbcd328eccd19fc45 2024-11-11T16:16:53,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/e94afad066fb4c22bb33bce9bee9897d is 50, key is test_row_0/B:col10/1731341812189/Put/seqid=0 2024-11-11T16:16:53,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742437_1613 (size=12151) 2024-11-11T16:16:53,601 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/e94afad066fb4c22bb33bce9bee9897d 2024-11-11T16:16:53,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/cb4e9b9acc5045e59631b0d9e8fe0186 is 50, key is test_row_0/C:col10/1731341812189/Put/seqid=0 2024-11-11T16:16:53,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742438_1614 (size=12151) 2024-11-11T16:16:53,617 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/cb4e9b9acc5045e59631b0d9e8fe0186 2024-11-11T16:16:53,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a74f3041bd6c415cbcd328eccd19fc45 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a74f3041bd6c415cbcd328eccd19fc45 2024-11-11T16:16:53,633 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a74f3041bd6c415cbcd328eccd19fc45, entries=150, sequenceid=169, filesize=30.4 K 2024-11-11T16:16:53,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/e94afad066fb4c22bb33bce9bee9897d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/e94afad066fb4c22bb33bce9bee9897d 2024-11-11T16:16:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,639 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/e94afad066fb4c22bb33bce9bee9897d, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T16:16:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/cb4e9b9acc5045e59631b0d9e8fe0186 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cb4e9b9acc5045e59631b0d9e8fe0186 2024-11-11T16:16:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,643 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cb4e9b9acc5045e59631b0d9e8fe0186, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T16:16:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,644 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 70a39dbdbb2217d780a9c88575e9f99d in 106ms, sequenceid=169, compaction requested=true 2024-11-11T16:16:53,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:53,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:53,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T16:16:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T16:16:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T16:16:53,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5140 sec 2024-11-11T16:16:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.5230 sec 2024-11-11T16:16:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T16:16:54,276 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T16:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,279 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T16:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,280 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:54,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T16:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:54,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:54,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:54,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:54,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:54,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T16:16:54,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111814ceb3e2e54f088a73fb865fbd2b4e_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341814383/Put/seqid=0 2024-11-11T16:16:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742440_1616 (size=24758) 2024-11-11T16:16:54,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341814398, max timestamp=1731341814421Num uniques 859; 2024-11-11T16:16:54,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:54,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341814398, max timestamp=1731341814421Num uniques 859; 2024-11-11T16:16:54,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:54,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731341814398, max timestamp=1731341814421Num uniques 859; 2024-11-11T16:16:54,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T16:16:54,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341874451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341874452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341874452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341874453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341874566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341874566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341874566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341874566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T16:16:54,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:54,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:54,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341874776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341874776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341874776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341874776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:54,825 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:54,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111814ceb3e2e54f088a73fb865fbd2b4e_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111814ceb3e2e54f088a73fb865fbd2b4e_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:54,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a85db3b56a694a049547beb212e5399f, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:54,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a85db3b56a694a049547beb212e5399f is 175, key is test_row_0/A:col10/1731341814383/Put/seqid=0 2024-11-11T16:16:54,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742439_1615 (size=74395) 2024-11-11T16:16:54,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:54,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T16:16:55,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341875085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341875086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341875086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341875086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341875226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,230 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 70a39dbdbb2217d780a9c88575e9f99d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a85db3b56a694a049547beb212e5399f 2024-11-11T16:16:55,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/192ddf8efdca40a1973ec9f3c8622b6e is 50, key is test_row_0/B:col10/1731341814383/Put/seqid=0 2024-11-11T16:16:55,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742441_1617 (size=12151) 2024-11-11T16:16:55,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T16:16:55,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:55,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:55,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341875596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341875596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341875606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341875606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:55,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/192ddf8efdca40a1973ec9f3c8622b6e 2024-11-11T16:16:55,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/c191cf7e8f414aa69eba8548a5ed7664 is 50, key is test_row_0/C:col10/1731341814383/Put/seqid=0 2024-11-11T16:16:55,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:55,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742442_1618 (size=12151) 2024-11-11T16:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:55,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/c191cf7e8f414aa69eba8548a5ed7664 2024-11-11T16:16:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/a85db3b56a694a049547beb212e5399f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a85db3b56a694a049547beb212e5399f 2024-11-11T16:16:55,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a85db3b56a694a049547beb212e5399f, entries=400, sequenceid=180, filesize=72.7 K 2024-11-11T16:16:55,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/192ddf8efdca40a1973ec9f3c8622b6e as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/192ddf8efdca40a1973ec9f3c8622b6e 2024-11-11T16:16:55,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/192ddf8efdca40a1973ec9f3c8622b6e, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T16:16:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/c191cf7e8f414aa69eba8548a5ed7664 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c191cf7e8f414aa69eba8548a5ed7664 2024-11-11T16:16:55,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c191cf7e8f414aa69eba8548a5ed7664, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T16:16:55,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 70a39dbdbb2217d780a9c88575e9f99d in 1306ms, sequenceid=180, compaction requested=true 2024-11-11T16:16:55,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:55,691 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:55,693 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:55,694 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176628 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:55,694 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/A is initiating minor compaction (all files) 2024-11-11T16:16:55,694 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/A in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,694 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/d6848ad377f34ad3bb39855c93119f91, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/e558016fb16f4835825da9f86d4b2aac, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a74f3041bd6c415cbcd328eccd19fc45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a85db3b56a694a049547beb212e5399f] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=172.5 K 2024-11-11T16:16:55,694 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,694 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/d6848ad377f34ad3bb39855c93119f91, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/e558016fb16f4835825da9f86d4b2aac, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a74f3041bd6c415cbcd328eccd19fc45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a85db3b56a694a049547beb212e5399f] 2024-11-11T16:16:55,695 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6848ad377f34ad3bb39855c93119f91, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341810655 2024-11-11T16:16:55,697 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:55,697 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/B is initiating minor compaction (all files) 2024-11-11T16:16:55,697 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/B in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,697 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/708d53b81895491e987f4276eb6abdde, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4008049363c84e9d81f9dfe8b761c50e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/e94afad066fb4c22bb33bce9bee9897d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/192ddf8efdca40a1973ec9f3c8622b6e] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=47.7 K 2024-11-11T16:16:55,697 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting e558016fb16f4835825da9f86d4b2aac, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731341811018 2024-11-11T16:16:55,697 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 708d53b81895491e987f4276eb6abdde, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341810655 2024-11-11T16:16:55,697 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74f3041bd6c415cbcd328eccd19fc45, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731341812178 2024-11-11T16:16:55,698 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4008049363c84e9d81f9dfe8b761c50e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731341811018 2024-11-11T16:16:55,698 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85db3b56a694a049547beb212e5399f, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731341814363 2024-11-11T16:16:55,698 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting e94afad066fb4c22bb33bce9bee9897d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731341812178 2024-11-11T16:16:55,700 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 192ddf8efdca40a1973ec9f3c8622b6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731341814376 2024-11-11T16:16:55,709 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:55,719 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#B#compaction#526 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:55,720 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/96b2a28b75ea499eb2d1015d639b13bb is 50, key is test_row_0/B:col10/1731341814383/Put/seqid=0 2024-11-11T16:16:55,725 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111133341b49fca24f2c8f289cbfb170c0b0_70a39dbdbb2217d780a9c88575e9f99d store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:55,729 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111133341b49fca24f2c8f289cbfb170c0b0_70a39dbdbb2217d780a9c88575e9f99d, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:55,729 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111133341b49fca24f2c8f289cbfb170c0b0_70a39dbdbb2217d780a9c88575e9f99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:55,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742444_1620 (size=4469) 2024-11-11T16:16:55,761 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#A#compaction#525 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:55,762 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/18b328d0c1724e00b56a34cf9a2169a3 is 175, key is test_row_0/A:col10/1731341814383/Put/seqid=0 2024-11-11T16:16:55,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742443_1619 (size=12561) 2024-11-11T16:16:55,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742445_1621 (size=31515) 2024-11-11T16:16:55,799 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/18b328d0c1724e00b56a34cf9a2169a3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/18b328d0c1724e00b56a34cf9a2169a3 2024-11-11T16:16:55,803 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/A of 70a39dbdbb2217d780a9c88575e9f99d into 18b328d0c1724e00b56a34cf9a2169a3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:55,803 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:55,803 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/A, priority=12, startTime=1731341815691; duration=0sec 2024-11-11T16:16:55,804 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:55,804 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:A 2024-11-11T16:16:55,804 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T16:16:55,804 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T16:16:55,804 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/C is initiating minor compaction (all files) 2024-11-11T16:16:55,805 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/C in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,805 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/a1614495616b406f8ffbeb8986d198b8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6c109f1be89645b0bbe25ec6a69e38f8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cb4e9b9acc5045e59631b0d9e8fe0186, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c191cf7e8f414aa69eba8548a5ed7664] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=47.7 K 2024-11-11T16:16:55,805 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1614495616b406f8ffbeb8986d198b8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731341810655 2024-11-11T16:16:55,805 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c109f1be89645b0bbe25ec6a69e38f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731341811018 2024-11-11T16:16:55,805 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb4e9b9acc5045e59631b0d9e8fe0186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731341812178 2024-11-11T16:16:55,805 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting c191cf7e8f414aa69eba8548a5ed7664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731341814376 2024-11-11T16:16:55,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T16:16:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:55,816 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-11T16:16:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:55,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:55,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:55,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:55,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:55,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:55,819 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#C#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:55,820 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/df782852f2414d378584598d116f79b7 is 50, key is test_row_0/C:col10/1731341814383/Put/seqid=0 2024-11-11T16:16:55,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742446_1622 (size=12561) 2024-11-11T16:16:55,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111107bee693c8194e088d9ddf76eee3eee4_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341814450/Put/seqid=0 2024-11-11T16:16:55,847 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/df782852f2414d378584598d116f79b7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/df782852f2414d378584598d116f79b7 2024-11-11T16:16:55,851 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/C of 70a39dbdbb2217d780a9c88575e9f99d into df782852f2414d378584598d116f79b7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:55,851 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:55,851 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/C, priority=12, startTime=1731341815693; duration=0sec 2024-11-11T16:16:55,851 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:55,851 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:C 2024-11-11T16:16:55,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742447_1623 (size=12304) 2024-11-11T16:16:56,170 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/96b2a28b75ea499eb2d1015d639b13bb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/96b2a28b75ea499eb2d1015d639b13bb 2024-11-11T16:16:56,175 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/B of 70a39dbdbb2217d780a9c88575e9f99d into 96b2a28b75ea499eb2d1015d639b13bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:56,175 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:56,175 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/B, priority=12, startTime=1731341815692; duration=0sec 2024-11-11T16:16:56,175 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:56,175 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:B 2024-11-11T16:16:56,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:56,259 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111107bee693c8194e088d9ddf76eee3eee4_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111107bee693c8194e088d9ddf76eee3eee4_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43736b8b668147eaa97ebe2c36afde23, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43736b8b668147eaa97ebe2c36afde23 is 175, key is test_row_0/A:col10/1731341814450/Put/seqid=0 2024-11-11T16:16:56,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742448_1624 (size=31105) 2024-11-11T16:16:56,264 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43736b8b668147eaa97ebe2c36afde23 2024-11-11T16:16:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/acbc9bff2008484ab3afde7e4deea01f is 50, key is test_row_0/B:col10/1731341814450/Put/seqid=0 2024-11-11T16:16:56,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742449_1625 (size=12151) 2024-11-11T16:16:56,293 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/acbc9bff2008484ab3afde7e4deea01f 2024-11-11T16:16:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/f9af84299ce44772b8a7efbe71b7fac3 is 50, key is test_row_0/C:col10/1731341814450/Put/seqid=0 2024-11-11T16:16:56,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742450_1626 (size=12151) 2024-11-11T16:16:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T16:16:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:56,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:56,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341876631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341876637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341876639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341876640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,725 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/f9af84299ce44772b8a7efbe71b7fac3 2024-11-11T16:16:56,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43736b8b668147eaa97ebe2c36afde23 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43736b8b668147eaa97ebe2c36afde23 2024-11-11T16:16:56,735 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43736b8b668147eaa97ebe2c36afde23, entries=150, sequenceid=215, filesize=30.4 K 2024-11-11T16:16:56,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/acbc9bff2008484ab3afde7e4deea01f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/acbc9bff2008484ab3afde7e4deea01f 2024-11-11T16:16:56,741 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/acbc9bff2008484ab3afde7e4deea01f, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T16:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/f9af84299ce44772b8a7efbe71b7fac3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/f9af84299ce44772b8a7efbe71b7fac3 2024-11-11T16:16:56,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341876747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,749 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/f9af84299ce44772b8a7efbe71b7fac3, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T16:16:56,750 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 70a39dbdbb2217d780a9c88575e9f99d in 934ms, sequenceid=215, compaction requested=false 2024-11-11T16:16:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T16:16:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T16:16:56,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T16:16:56,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4720 sec 2024-11-11T16:16:56,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.4780 sec 2024-11-11T16:16:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:56,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T16:16:56,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:56,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:56,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:56,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:56,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:56,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:16:56,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111205d8845e0fb4cf5a3645043563bfb28_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341816756/Put/seqid=0 2024-11-11T16:16:56,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742451_1627 (size=12304) 2024-11-11T16:16:56,771 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:56,774 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111205d8845e0fb4cf5a3645043563bfb28_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111205d8845e0fb4cf5a3645043563bfb28_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:56,776 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43f1b6a6764741098d48ae67fefbba14, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:56,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43f1b6a6764741098d48ae67fefbba14 is 175, key is test_row_0/A:col10/1731341816756/Put/seqid=0 2024-11-11T16:16:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742452_1628 (size=31105) 2024-11-11T16:16:56,782 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43f1b6a6764741098d48ae67fefbba14 2024-11-11T16:16:56,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/b7b5a6a1ab494989acdfeb8b8e5c2922 is 50, key is test_row_0/B:col10/1731341816756/Put/seqid=0 2024-11-11T16:16:56,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341876793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742453_1629 (size=12151) 2024-11-11T16:16:56,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/b7b5a6a1ab494989acdfeb8b8e5c2922 2024-11-11T16:16:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341876797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341876798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/cfd0090fe7434bfdb792fd037e721e8a is 50, key is test_row_0/C:col10/1731341816756/Put/seqid=0 2024-11-11T16:16:56,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742454_1630 (size=12151) 2024-11-11T16:16:56,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341876906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341876906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341876916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:56,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341876955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341877116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341877116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341877129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/cfd0090fe7434bfdb792fd037e721e8a 2024-11-11T16:16:57,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/43f1b6a6764741098d48ae67fefbba14 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43f1b6a6764741098d48ae67fefbba14 2024-11-11T16:16:57,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43f1b6a6764741098d48ae67fefbba14, entries=150, sequenceid=229, filesize=30.4 K 2024-11-11T16:16:57,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/b7b5a6a1ab494989acdfeb8b8e5c2922 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/b7b5a6a1ab494989acdfeb8b8e5c2922 2024-11-11T16:16:57,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/b7b5a6a1ab494989acdfeb8b8e5c2922, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T16:16:57,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/cfd0090fe7434bfdb792fd037e721e8a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cfd0090fe7434bfdb792fd037e721e8a 2024-11-11T16:16:57,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cfd0090fe7434bfdb792fd037e721e8a, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T16:16:57,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 70a39dbdbb2217d780a9c88575e9f99d in 478ms, sequenceid=229, compaction requested=true 2024-11-11T16:16:57,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:57,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:57,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:57,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:57,235 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:57,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T16:16:57,236 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:57,236 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:57,237 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/A is initiating minor compaction (all files) 2024-11-11T16:16:57,237 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/A in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:57,237 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:57,237 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/B is initiating minor compaction (all files) 2024-11-11T16:16:57,237 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/18b328d0c1724e00b56a34cf9a2169a3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43736b8b668147eaa97ebe2c36afde23, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43f1b6a6764741098d48ae67fefbba14] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=91.5 K 2024-11-11T16:16:57,237 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/B in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:57,237 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:57,237 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/96b2a28b75ea499eb2d1015d639b13bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/acbc9bff2008484ab3afde7e4deea01f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/b7b5a6a1ab494989acdfeb8b8e5c2922] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.0 K 2024-11-11T16:16:57,237 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/18b328d0c1724e00b56a34cf9a2169a3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43736b8b668147eaa97ebe2c36afde23, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43f1b6a6764741098d48ae67fefbba14] 2024-11-11T16:16:57,237 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b328d0c1724e00b56a34cf9a2169a3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731341814376 2024-11-11T16:16:57,238 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43736b8b668147eaa97ebe2c36afde23, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731341814450 2024-11-11T16:16:57,238 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43f1b6a6764741098d48ae67fefbba14, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731341816631 2024-11-11T16:16:57,238 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b2a28b75ea499eb2d1015d639b13bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731341814376 2024-11-11T16:16:57,240 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting acbc9bff2008484ab3afde7e4deea01f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731341814450 2024-11-11T16:16:57,240 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b5a6a1ab494989acdfeb8b8e5c2922, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731341816631 2024-11-11T16:16:57,246 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:57,254 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111793ea44de39c4ae8bf95ef7d8b9d50d5_70a39dbdbb2217d780a9c88575e9f99d store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:57,256 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111793ea44de39c4ae8bf95ef7d8b9d50d5_70a39dbdbb2217d780a9c88575e9f99d, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:57,256 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111793ea44de39c4ae8bf95ef7d8b9d50d5_70a39dbdbb2217d780a9c88575e9f99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:57,259 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#B#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:57,259 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/c08e45c2aaa74ac5927033529e61fc8d is 50, key is test_row_0/B:col10/1731341816756/Put/seqid=0 2024-11-11T16:16:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:57,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341816791, max timestamp=1731341817270Num uniques 859; 2024-11-11T16:16:57,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:57,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341816791, max timestamp=1731341817270Num uniques 859; 2024-11-11T16:16:57,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:57,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341816791, max timestamp=1731341817270Num uniques 859; 2024-11-11T16:16:57,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:57,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-11T16:16:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:57,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742456_1632 (size=12663) 2024-11-11T16:16:57,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742455_1631 (size=4469) 2024-11-11T16:16:57,282 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#A#compaction#534 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:57,283 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/674132344b4a4cd9b4ad8a1803d3b927 is 175, key is test_row_0/A:col10/1731341816756/Put/seqid=0 2024-11-11T16:16:57,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111173d198777eec4a5bb7d33cdd82c6a506_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341817270/Put/seqid=0 2024-11-11T16:16:57,287 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/c08e45c2aaa74ac5927033529e61fc8d as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/c08e45c2aaa74ac5927033529e61fc8d 2024-11-11T16:16:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742457_1633 (size=31617) 2024-11-11T16:16:57,293 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/B of 70a39dbdbb2217d780a9c88575e9f99d into c08e45c2aaa74ac5927033529e61fc8d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:57,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:57,293 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/B, priority=13, startTime=1731341817235; duration=0sec 2024-11-11T16:16:57,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:57,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:B 2024-11-11T16:16:57,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:57,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742458_1634 (size=14794) 2024-11-11T16:16:57,301 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:57,301 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/C is initiating minor compaction (all files) 2024-11-11T16:16:57,301 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/C in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:57,301 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/df782852f2414d378584598d116f79b7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/f9af84299ce44772b8a7efbe71b7fac3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cfd0090fe7434bfdb792fd037e721e8a] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.0 K 2024-11-11T16:16:57,301 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:57,301 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting df782852f2414d378584598d116f79b7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731341814376 2024-11-11T16:16:57,302 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting f9af84299ce44772b8a7efbe71b7fac3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731341814450 2024-11-11T16:16:57,302 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting cfd0090fe7434bfdb792fd037e721e8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731341816631 2024-11-11T16:16:57,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111173d198777eec4a5bb7d33cdd82c6a506_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173d198777eec4a5bb7d33cdd82c6a506_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:57,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/3bf653a3fe37496f899c48251fbd6374, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:57,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/3bf653a3fe37496f899c48251fbd6374 is 175, key is test_row_0/A:col10/1731341817270/Put/seqid=0 2024-11-11T16:16:57,312 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#C#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:57,312 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/6a2b823fffe4462d988f9b5ac552e2c8 is 50, key is test_row_0/C:col10/1731341816756/Put/seqid=0 2024-11-11T16:16:57,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742459_1635 (size=39749) 2024-11-11T16:16:57,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/3bf653a3fe37496f899c48251fbd6374 2024-11-11T16:16:57,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742460_1636 (size=12663) 2024-11-11T16:16:57,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/4a5848cbf6c54c528fa7ac8f17a04b4a is 50, key is test_row_0/B:col10/1731341817270/Put/seqid=0 2024-11-11T16:16:57,329 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/6a2b823fffe4462d988f9b5ac552e2c8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6a2b823fffe4462d988f9b5ac552e2c8 2024-11-11T16:16:57,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742461_1637 (size=12151) 2024-11-11T16:16:57,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/4a5848cbf6c54c528fa7ac8f17a04b4a 2024-11-11T16:16:57,335 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/C of 70a39dbdbb2217d780a9c88575e9f99d into 6a2b823fffe4462d988f9b5ac552e2c8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:57,335 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:57,336 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/C, priority=13, startTime=1731341817235; duration=0sec 2024-11-11T16:16:57,336 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:57,336 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:C 2024-11-11T16:16:57,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/03babbe272544a5b8a65ede5ce13fe2a is 50, key is test_row_0/C:col10/1731341817270/Put/seqid=0 2024-11-11T16:16:57,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742462_1638 (size=12151) 2024-11-11T16:16:57,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341817346, max timestamp=1731341817369Num uniques 859; 2024-11-11T16:16:57,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:57,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341817346, max timestamp=1731341817369Num uniques 859; 2024-11-11T16:16:57,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:57,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341817346, max timestamp=1731341817369Num uniques 859; 2024-11-11T16:16:57,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:57,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341817434, max timestamp=1731341817441Num uniques 1018; 2024-11-11T16:16:57,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T16:16:57,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341817434, max timestamp=1731341817441Num uniques 1018; 2024-11-11T16:16:57,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T16:16:57,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341817434, max timestamp=1731341817441Num uniques 1018; 2024-11-11T16:16:57,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T16:16:57,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341817461, max timestamp=1731341817475Num uniques 1027; 2024-11-11T16:16:57,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T16:16:57,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341817461, max timestamp=1731341817475Num uniques 1027; 2024-11-11T16:16:57,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T16:16:57,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731341817461, max timestamp=1731341817475Num uniques 1027; 2024-11-11T16:16:57,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T16:16:57,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10801, totalHeapSize=24784, min timestamp=1731341817491, max timestamp=1731341817509Num uniques 1036; 2024-11-11T16:16:57,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47441, new segment data size=10801, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-11T16:16:57,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1731341817491, max timestamp=1731341817509Num uniques 1036; 2024-11-11T16:16:57,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=10796, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-11T16:16:57,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10804, totalHeapSize=24784, min timestamp=1731341817491, max timestamp=1731341817509Num uniques 1036; 2024-11-11T16:16:57,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47444, new segment data size=10804, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-11T16:16:57,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341817520, max timestamp=1731341817543Num uniques 1095; 2024-11-11T16:16:57,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-11T16:16:57,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341817520, max timestamp=1731341817543Num uniques 1095; 2024-11-11T16:16:57,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-11T16:16:57,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341817520, max timestamp=1731341817543Num uniques 1095; 2024-11-11T16:16:57,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-11T16:16:57,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731341817604, max timestamp=1731341817621Num uniques 1054; 2024-11-11T16:16:57,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-11T16:16:57,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731341817604, max timestamp=1731341817621Num uniques 1054; 2024-11-11T16:16:57,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-11T16:16:57,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731341817604, max timestamp=1731341817621Num uniques 1054; 2024-11-11T16:16:57,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-11T16:16:57,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7455, totalHeapSize=17192, min timestamp=1731341817650, max timestamp=1731341817659Num uniques 1113; 2024-11-11T16:16:57,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50965, new segment data size=7455, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-11T16:16:57,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731341817650, max timestamp=1731341817658Num uniques 1113; 2024-11-11T16:16:57,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-11T16:16:57,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7455, totalHeapSize=17192, min timestamp=1731341817650, max timestamp=1731341817659Num uniques 1113; 2024-11-11T16:16:57,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50965, new segment data size=7455, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-11T16:16:57,694 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/674132344b4a4cd9b4ad8a1803d3b927 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/674132344b4a4cd9b4ad8a1803d3b927 2024-11-11T16:16:57,698 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/A of 70a39dbdbb2217d780a9c88575e9f99d into 674132344b4a4cd9b4ad8a1803d3b927(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:57,698 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:57,699 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/A, priority=13, startTime=1731341817235; duration=0sec 2024-11-11T16:16:57,699 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:57,699 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:A 2024-11-11T16:16:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731341817681, max timestamp=1731341817697Num uniques 1022; 2024-11-11T16:16:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-11T16:16:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731341817681, max timestamp=1731341817697Num uniques 1022; 2024-11-11T16:16:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-11T16:16:57,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731341817681, max timestamp=1731341817697Num uniques 1022; 2024-11-11T16:16:57,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-11T16:16:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731341817720, max timestamp=1731341817731Num uniques 1081; 2024-11-11T16:16:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731341817720, max timestamp=1731341817731Num uniques 1081; 2024-11-11T16:16:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=8276, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-11T16:16:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8279, totalHeapSize=19064, min timestamp=1731341817720, max timestamp=1731341817731Num uniques 1081; 2024-11-11T16:16:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=8276, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-11T16:16:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49499, new segment data size=8279, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-11T16:16:57,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/03babbe272544a5b8a65ede5ce13fe2a 2024-11-11T16:16:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/3bf653a3fe37496f899c48251fbd6374 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/3bf653a3fe37496f899c48251fbd6374 2024-11-11T16:16:57,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/3bf653a3fe37496f899c48251fbd6374, entries=200, sequenceid=250, filesize=38.8 K 2024-11-11T16:16:57,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/4a5848cbf6c54c528fa7ac8f17a04b4a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4a5848cbf6c54c528fa7ac8f17a04b4a 2024-11-11T16:16:57,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4a5848cbf6c54c528fa7ac8f17a04b4a, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T16:16:57,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/03babbe272544a5b8a65ede5ce13fe2a as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/03babbe272544a5b8a65ede5ce13fe2a 2024-11-11T16:16:57,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/03babbe272544a5b8a65ede5ce13fe2a, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T16:16:57,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=73.80 KB/75570 for 70a39dbdbb2217d780a9c88575e9f99d in 482ms, sequenceid=250, compaction requested=false 2024-11-11T16:16:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:57,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=80.51 KB heapSize=223.55 KB 2024-11-11T16:16:57,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:57,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111102c52acf820341ca9bc9928f632bc4f8_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341817744/Put/seqid=0 2024-11-11T16:16:57,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742463_1639 (size=14994) 2024-11-11T16:16:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341877784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341877787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341877788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341877790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341877895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341877895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341877896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:57,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341877906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341878106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341878106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341878106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341878116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:58,180 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111102c52acf820341ca9bc9928f632bc4f8_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102c52acf820341ca9bc9928f632bc4f8_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:58,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/1479fd91c8304e20b2225b0e476654a3, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:58,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/1479fd91c8304e20b2225b0e476654a3 is 175, key is test_row_0/A:col10/1731341817744/Put/seqid=0 2024-11-11T16:16:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742464_1640 (size=39949) 2024-11-11T16:16:58,191 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/1479fd91c8304e20b2225b0e476654a3 2024-11-11T16:16:58,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/38c9e756733946159acdb218c6e2a720 is 50, key is test_row_0/B:col10/1731341817744/Put/seqid=0 2024-11-11T16:16:58,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742465_1641 (size=12301) 2024-11-11T16:16:58,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/38c9e756733946159acdb218c6e2a720 2024-11-11T16:16:58,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/4586b4b62ff749ddba1fd23c745cd327 is 50, key is test_row_0/C:col10/1731341817744/Put/seqid=0 2024-11-11T16:16:58,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742466_1642 (size=12301) 2024-11-11T16:16:58,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/4586b4b62ff749ddba1fd23c745cd327 2024-11-11T16:16:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/1479fd91c8304e20b2225b0e476654a3 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/1479fd91c8304e20b2225b0e476654a3 2024-11-11T16:16:58,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/1479fd91c8304e20b2225b0e476654a3, entries=200, sequenceid=419, filesize=39.0 K 2024-11-11T16:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/38c9e756733946159acdb218c6e2a720 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/38c9e756733946159acdb218c6e2a720 2024-11-11T16:16:58,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/38c9e756733946159acdb218c6e2a720, entries=150, sequenceid=419, filesize=12.0 K 2024-11-11T16:16:58,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/4586b4b62ff749ddba1fd23c745cd327 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4586b4b62ff749ddba1fd23c745cd327 2024-11-11T16:16:58,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4586b4b62ff749ddba1fd23c745cd327, entries=150, sequenceid=419, filesize=12.0 K 2024-11-11T16:16:58,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~223.50 KB/228864, currentSize=100.63 KB/103050 for 70a39dbdbb2217d780a9c88575e9f99d in 489ms, sequenceid=419, compaction requested=true 2024-11-11T16:16:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:16:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:58,250 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:16:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:16:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:58,250 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:58,251 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:58,251 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:58,251 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/B is initiating minor compaction (all files) 2024-11-11T16:16:58,251 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/A is initiating minor compaction (all files) 2024-11-11T16:16:58,251 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/B in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,251 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/A in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,251 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/c08e45c2aaa74ac5927033529e61fc8d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4a5848cbf6c54c528fa7ac8f17a04b4a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/38c9e756733946159acdb218c6e2a720] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.2 K 2024-11-11T16:16:58,251 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/674132344b4a4cd9b4ad8a1803d3b927, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/3bf653a3fe37496f899c48251fbd6374, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/1479fd91c8304e20b2225b0e476654a3] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=108.7 K 2024-11-11T16:16:58,251 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,252 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/674132344b4a4cd9b4ad8a1803d3b927, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/3bf653a3fe37496f899c48251fbd6374, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/1479fd91c8304e20b2225b0e476654a3] 2024-11-11T16:16:58,252 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c08e45c2aaa74ac5927033529e61fc8d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731341816631 2024-11-11T16:16:58,252 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 674132344b4a4cd9b4ad8a1803d3b927, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731341816631 2024-11-11T16:16:58,252 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a5848cbf6c54c528fa7ac8f17a04b4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731341816796 2024-11-11T16:16:58,253 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf653a3fe37496f899c48251fbd6374, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731341816791 2024-11-11T16:16:58,253 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c9e756733946159acdb218c6e2a720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731341817744 2024-11-11T16:16:58,253 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1479fd91c8304e20b2225b0e476654a3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731341817742 2024-11-11T16:16:58,259 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:58,260 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#B#compaction#544 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:58,261 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112d4b74f25e534473ba9cc681ebf80c3e_70a39dbdbb2217d780a9c88575e9f99d store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:58,261 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/3b04732c32644c87bb123562f9c0351f is 50, key is test_row_0/B:col10/1731341817744/Put/seqid=0 2024-11-11T16:16:58,263 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112d4b74f25e534473ba9cc681ebf80c3e_70a39dbdbb2217d780a9c88575e9f99d, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:58,263 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112d4b74f25e534473ba9cc681ebf80c3e_70a39dbdbb2217d780a9c88575e9f99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:58,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742467_1643 (size=12915) 2024-11-11T16:16:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742468_1644 (size=4469) 2024-11-11T16:16:58,286 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/3b04732c32644c87bb123562f9c0351f as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3b04732c32644c87bb123562f9c0351f 2024-11-11T16:16:58,293 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/B of 70a39dbdbb2217d780a9c88575e9f99d into 3b04732c32644c87bb123562f9c0351f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:58,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:58,293 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/B, priority=13, startTime=1731341818250; duration=0sec 2024-11-11T16:16:58,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:16:58,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:B 2024-11-11T16:16:58,293 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:16:58,294 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:16:58,294 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/C is initiating minor compaction (all files) 2024-11-11T16:16:58,294 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/C in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,294 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6a2b823fffe4462d988f9b5ac552e2c8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/03babbe272544a5b8a65ede5ce13fe2a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4586b4b62ff749ddba1fd23c745cd327] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.2 K 2024-11-11T16:16:58,295 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2b823fffe4462d988f9b5ac552e2c8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731341816631 2024-11-11T16:16:58,295 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 03babbe272544a5b8a65ede5ce13fe2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731341816796 2024-11-11T16:16:58,295 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4586b4b62ff749ddba1fd23c745cd327, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731341817744 2024-11-11T16:16:58,318 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#C#compaction#545 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:58,319 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/af5e8784063844679fd5f77a92cc0293 is 50, key is test_row_0/C:col10/1731341817744/Put/seqid=0 2024-11-11T16:16:58,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742469_1645 (size=12915) 2024-11-11T16:16:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:58,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T16:16:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341817785, max timestamp=1731341818419Num uniques 859; 2024-11-11T16:16:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341817785, max timestamp=1731341818419Num uniques 859; 2024-11-11T16:16:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731341817785, max timestamp=1731341818419Num uniques 859; 2024-11-11T16:16:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T16:16:58,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:58,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110509f5255f4410db312a06ffcf8359e_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341818419/Put/seqid=0 2024-11-11T16:16:58,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742470_1646 (size=14994) 2024-11-11T16:16:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T16:16:58,456 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:16:58,456 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:16:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T16:16:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T16:16:58,458 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:16:58,458 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:16:58,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:16:58,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341818443, max timestamp=1731341818461Num uniques 859; 2024-11-11T16:16:58,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:58,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341818443, max timestamp=1731341818461Num uniques 859; 2024-11-11T16:16:58,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:58,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731341818443, max timestamp=1731341818461Num uniques 859; 2024-11-11T16:16:58,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T16:16:58,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341818483, max timestamp=1731341818511Num uniques 1068; 2024-11-11T16:16:58,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T16:16:58,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341818483, max timestamp=1731341818511Num uniques 1068; 2024-11-11T16:16:58,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T16:16:58,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731341818483, max timestamp=1731341818511Num uniques 1068; 2024-11-11T16:16:58,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T16:16:58,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341818542, max timestamp=1731341818562Num uniques 1027; 2024-11-11T16:16:58,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341818542, max timestamp=1731341818562Num uniques 1027; 2024-11-11T16:16:58,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T16:16:58,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T16:16:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T16:16:58,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731341818542, max timestamp=1731341818562Num uniques 1027; 2024-11-11T16:16:58,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T16:16:58,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731341818597, max timestamp=1731341818618Num uniques 1086; 2024-11-11T16:16:58,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T16:16:58,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731341818597, max timestamp=1731341818618Num uniques 1086; 2024-11-11T16:16:58,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T16:16:58,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731341818597, max timestamp=1731341818618Num uniques 1086; 2024-11-11T16:16:58,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-11T16:16:58,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341818661, max timestamp=1731341818672Num uniques 1095; 2024-11-11T16:16:58,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-11T16:16:58,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731341818661, max timestamp=1731341818672Num uniques 1095; 2024-11-11T16:16:58,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-11T16:16:58,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341818661, max timestamp=1731341818672Num uniques 1095; 2024-11-11T16:16:58,680 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#A#compaction#543 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:16:58,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-11-11T16:16:58,681 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/9652e5f856474eeb9338e462e143d10c is 175, key is test_row_0/A:col10/1731341817744/Put/seqid=0 2024-11-11T16:16:58,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742471_1647 (size=31869) 2024-11-11T16:16:58,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731341818702, max timestamp=1731341818723Num uniques 1009; 2024-11-11T16:16:58,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=11630, suffix heap size=132904,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=254 2024-11-11T16:16:58,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11624, totalHeapSize=26656, min timestamp=1731341818702, max timestamp=1731341818723Num uniques 1054; 2024-11-11T16:16:58,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48264, new segment data size=11624, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-11T16:16:58,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731341818702, max timestamp=1731341818723Num uniques 1054; 2024-11-11T16:16:58,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-11T16:16:58,757 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/af5e8784063844679fd5f77a92cc0293 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af5e8784063844679fd5f77a92cc0293 2024-11-11T16:16:58,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,768 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/C of 70a39dbdbb2217d780a9c88575e9f99d into af5e8784063844679fd5f77a92cc0293(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:58,768 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:58,768 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/C, priority=13, startTime=1731341818250; duration=0sec 2024-11-11T16:16:58,769 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:58,769 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:C 2024-11-11T16:16:58,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731341818747, max timestamp=1731341818773Num uniques 1113; 2024-11-11T16:16:58,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-11T16:16:58,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731341818747, max timestamp=1731341818773Num uniques 1113; 2024-11-11T16:16:58,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-11T16:16:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T16:16:58,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731341818747, max timestamp=1731341818773Num uniques 1113; 2024-11-11T16:16:58,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-11T16:16:58,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731341818805, max timestamp=1731341818817Num uniques 1072; 2024-11-11T16:16:58,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T16:16:58,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731341818805, max timestamp=1731341818817Num uniques 1072; 2024-11-11T16:16:58,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T16:16:58,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731341818805, max timestamp=1731341818817Num uniques 1072; 2024-11-11T16:16:58,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T16:16:58,840 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:58,843 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110509f5255f4410db312a06ffcf8359e_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110509f5255f4410db312a06ffcf8359e_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:58,844 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2e97381461fe4d73b7032651ee984a74, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:58,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2e97381461fe4d73b7032651ee984a74 is 175, key is test_row_0/A:col10/1731341818419/Put/seqid=0 2024-11-11T16:16:58,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742472_1648 (size=39949) 2024-11-11T16:16:58,849 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=441, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2e97381461fe4d73b7032651ee984a74 2024-11-11T16:16:58,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/a2bf42ad06c943f18d5943cae26f7bf5 is 50, key is test_row_0/B:col10/1731341818419/Put/seqid=0 2024-11-11T16:16:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731341818854, max timestamp=1731341818873Num uniques 1081; 2024-11-11T16:16:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T16:16:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731341818854, max timestamp=1731341818873Num uniques 1081; 2024-11-11T16:16:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T16:16:58,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731341818854, max timestamp=1731341818873Num uniques 1081; 2024-11-11T16:16:58,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T16:16:58,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742473_1649 (size=12301) 2024-11-11T16:16:58,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/a2bf42ad06c943f18d5943cae26f7bf5 2024-11-11T16:16:58,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/518f73011ec042e089a8a7dd3c70e91c is 50, key is test_row_0/C:col10/1731341818419/Put/seqid=0 2024-11-11T16:16:58,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742474_1650 (size=12301) 2024-11-11T16:16:58,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/518f73011ec042e089a8a7dd3c70e91c 2024-11-11T16:16:58,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:58,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:58,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:58,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10980, totalHeapSize=25200, min timestamp=1731341818903, max timestamp=1731341818925Num uniques 1090; 2024-11-11T16:16:58,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=10980, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-11T16:16:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/2e97381461fe4d73b7032651ee984a74 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2e97381461fe4d73b7032651ee984a74 2024-11-11T16:16:58,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2e97381461fe4d73b7032651ee984a74, entries=200, sequenceid=441, filesize=39.0 K 2024-11-11T16:16:58,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341818903, max timestamp=1731341818925Num uniques 1090; 2024-11-11T16:16:58,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/a2bf42ad06c943f18d5943cae26f7bf5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/a2bf42ad06c943f18d5943cae26f7bf5 2024-11-11T16:16:58,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=6870, suffix heap size=141328,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=150 2024-11-11T16:16:58,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731341818903, max timestamp=1731341818925Num uniques 1090; 2024-11-11T16:16:58,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/16b413a53992:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=6870, suffix heap size=141328,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=150 2024-11-11T16:16:58,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/a2bf42ad06c943f18d5943cae26f7bf5, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T16:16:58,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/518f73011ec042e089a8a7dd3c70e91c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/518f73011ec042e089a8a7dd3c70e91c 2024-11-11T16:16:58,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/518f73011ec042e089a8a7dd3c70e91c, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T16:16:58,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=79.19 KB/81090 for 70a39dbdbb2217d780a9c88575e9f99d in 533ms, sequenceid=441, compaction requested=false 2024-11-11T16:16:58,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:58,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=85.90 KB heapSize=239.58 KB 2024-11-11T16:16:58,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:16:58,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:58,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:16:58,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:58,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:16:58,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T16:16:58,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111787de8103a854ba79495a2e40cf78713_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341818952/Put/seqid=0 2024-11-11T16:16:58,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742475_1651 (size=12454) 2024-11-11T16:16:58,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341878976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341878977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341878983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341878983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:58,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T16:16:59,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341879086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341879096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341879096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341879096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,114 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/9652e5f856474eeb9338e462e143d10c as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/9652e5f856474eeb9338e462e143d10c 2024-11-11T16:16:59,119 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/A of 70a39dbdbb2217d780a9c88575e9f99d into 9652e5f856474eeb9338e462e143d10c(size=31.1 K), total size for store is 70.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:16:59,119 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:16:59,119 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/A, priority=13, startTime=1731341818250; duration=0sec 2024-11-11T16:16:59,119 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:16:59,119 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:A 2024-11-11T16:16:59,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57940 deadline: 1731341879255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,261 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 16b413a53992,45121,1731341648557 for 'test_row_1' in 70a39dbdbb2217d780a9c88575e9f99d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T16:16:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341879296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341879305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341879305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341879305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,368 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:16:59,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111787de8103a854ba79495a2e40cf78713_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111787de8103a854ba79495a2e40cf78713_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:16:59,372 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/94f787725f044aba9dbeca4236791bfe, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:16:59,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/94f787725f044aba9dbeca4236791bfe is 175, key is test_row_0/A:col10/1731341818952/Put/seqid=0 2024-11-11T16:16:59,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742476_1652 (size=31255) 2024-11-11T16:16:59,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T16:16:59,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341879609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341879616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341879616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:16:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341879616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:16:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:16:59,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:16:59,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:59,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:59,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,777 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=627, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/94f787725f044aba9dbeca4236791bfe 2024-11-11T16:16:59,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/d80a72a0f9634c0e83f48379ee046036 is 50, key is test_row_0/B:col10/1731341818952/Put/seqid=0 2024-11-11T16:16:59,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742477_1653 (size=12301) 2024-11-11T16:16:59,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:16:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:16:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:16:59,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:16:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:17:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341880120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341880126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341880126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341880126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:00,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:17:00,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.53 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/d80a72a0f9634c0e83f48379ee046036 2024-11-11T16:17:00,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/7753812b20fa4d89a940dd3c935a96e8 is 50, key is test_row_0/C:col10/1731341818952/Put/seqid=0 2024-11-11T16:17:00,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742478_1654 (size=12301) 2024-11-11T16:17:00,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:17:00,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:17:00,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T16:17:00,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:17:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:00,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.53 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/7753812b20fa4d89a940dd3c935a96e8 2024-11-11T16:17:00,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/94f787725f044aba9dbeca4236791bfe as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/94f787725f044aba9dbeca4236791bfe 2024-11-11T16:17:00,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/94f787725f044aba9dbeca4236791bfe, entries=150, sequenceid=627, filesize=30.5 K 2024-11-11T16:17:00,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/d80a72a0f9634c0e83f48379ee046036 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/d80a72a0f9634c0e83f48379ee046036 2024-11-11T16:17:00,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/d80a72a0f9634c0e83f48379ee046036, entries=150, sequenceid=627, filesize=12.0 K 2024-11-11T16:17:00,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/7753812b20fa4d89a940dd3c935a96e8 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7753812b20fa4d89a940dd3c935a96e8 2024-11-11T16:17:00,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7753812b20fa4d89a940dd3c935a96e8, entries=150, sequenceid=627, filesize=12.0 K 2024-11-11T16:17:00,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~92.61 KB/94830, heapSize ~259.45 KB/265680, currentSize=93.93 KB/96180 for 70a39dbdbb2217d780a9c88575e9f99d in 1760ms, sequenceid=627, compaction requested=true 2024-11-11T16:17:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:17:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:17:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:00,713 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:17:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:17:00,713 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:17:00,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:17:00,713 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:17:00,713 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:17:00,714 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/B is initiating minor compaction (all files) 2024-11-11T16:17:00,714 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/A is initiating minor compaction (all files) 2024-11-11T16:17:00,714 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/A in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,714 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/B in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,714 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/9652e5f856474eeb9338e462e143d10c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2e97381461fe4d73b7032651ee984a74, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/94f787725f044aba9dbeca4236791bfe] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=100.7 K 2024-11-11T16:17:00,714 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,714 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3b04732c32644c87bb123562f9c0351f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/a2bf42ad06c943f18d5943cae26f7bf5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/d80a72a0f9634c0e83f48379ee046036] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.6 K 2024-11-11T16:17:00,714 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/9652e5f856474eeb9338e462e143d10c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2e97381461fe4d73b7032651ee984a74, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/94f787725f044aba9dbeca4236791bfe] 2024-11-11T16:17:00,714 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b04732c32644c87bb123562f9c0351f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731341817744 2024-11-11T16:17:00,714 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9652e5f856474eeb9338e462e143d10c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731341817744 2024-11-11T16:17:00,714 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e97381461fe4d73b7032651ee984a74, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731341817785 2024-11-11T16:17:00,714 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting a2bf42ad06c943f18d5943cae26f7bf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731341817785 2024-11-11T16:17:00,715 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting d80a72a0f9634c0e83f48379ee046036, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731341818949 2024-11-11T16:17:00,715 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94f787725f044aba9dbeca4236791bfe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731341818949 2024-11-11T16:17:00,720 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:00,722 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#B#compaction#553 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:17:00,722 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e4606681106c4268a193b74eb8ecc1b2_70a39dbdbb2217d780a9c88575e9f99d store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:00,722 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/903e7dd5d39144ce90331fe4861f41e4 is 50, key is test_row_0/B:col10/1731341818952/Put/seqid=0 2024-11-11T16:17:00,724 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e4606681106c4268a193b74eb8ecc1b2_70a39dbdbb2217d780a9c88575e9f99d, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:00,724 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e4606681106c4268a193b74eb8ecc1b2_70a39dbdbb2217d780a9c88575e9f99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:00,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742479_1655 (size=13017) 2024-11-11T16:17:00,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742480_1656 (size=4469) 2024-11-11T16:17:00,729 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#A#compaction#552 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:17:00,729 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/8cce832f484c4950a967cde13ad3e008 is 175, key is test_row_0/A:col10/1731341818952/Put/seqid=0 2024-11-11T16:17:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742481_1657 (size=31971) 2024-11-11T16:17:00,736 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/8cce832f484c4950a967cde13ad3e008 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8cce832f484c4950a967cde13ad3e008 2024-11-11T16:17:00,741 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/A of 70a39dbdbb2217d780a9c88575e9f99d into 8cce832f484c4950a967cde13ad3e008(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:17:00,741 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:00,741 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/A, priority=13, startTime=1731341820713; duration=0sec 2024-11-11T16:17:00,742 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:17:00,742 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:A 2024-11-11T16:17:00,742 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:17:00,743 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:17:00,743 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/C is initiating minor compaction (all files) 2024-11-11T16:17:00,743 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/C in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,743 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af5e8784063844679fd5f77a92cc0293, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/518f73011ec042e089a8a7dd3c70e91c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7753812b20fa4d89a940dd3c935a96e8] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.6 K 2024-11-11T16:17:00,744 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting af5e8784063844679fd5f77a92cc0293, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731341817744 2024-11-11T16:17:00,744 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 518f73011ec042e089a8a7dd3c70e91c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731341817785 2024-11-11T16:17:00,745 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7753812b20fa4d89a940dd3c935a96e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731341818949 2024-11-11T16:17:00,751 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#C#compaction#554 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:17:00,751 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/c45d7d097eec459880808fa505e179a4 is 50, key is test_row_0/C:col10/1731341818952/Put/seqid=0 2024-11-11T16:17:00,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742482_1658 (size=13017) 2024-11-11T16:17:00,761 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/c45d7d097eec459880808fa505e179a4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c45d7d097eec459880808fa505e179a4 2024-11-11T16:17:00,768 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/C of 70a39dbdbb2217d780a9c88575e9f99d into c45d7d097eec459880808fa505e179a4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:17:00,768 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:00,768 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/C, priority=13, startTime=1731341820713; duration=0sec 2024-11-11T16:17:00,768 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:00,768 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:C 2024-11-11T16:17:00,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T16:17:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:00,794 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T16:17:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:17:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:17:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:17:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:00,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115007fb80eaec47d0925702893e3202a3_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341818982/Put/seqid=0 2024-11-11T16:17:00,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742483_1659 (size=12454) 2024-11-11T16:17:00,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:00,830 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115007fb80eaec47d0925702893e3202a3_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115007fb80eaec47d0925702893e3202a3_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/ec515547879b4264b59a4f264e7681ca, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/ec515547879b4264b59a4f264e7681ca is 175, key is test_row_0/A:col10/1731341818982/Put/seqid=0 2024-11-11T16:17:00,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742484_1660 (size=31255) 2024-11-11T16:17:00,838 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=647, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/ec515547879b4264b59a4f264e7681ca 2024-11-11T16:17:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/7130495acfa84fc8b0bd108f0b0d9aa0 is 50, key is test_row_0/B:col10/1731341818982/Put/seqid=0 2024-11-11T16:17:00,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742485_1661 (size=12301) 2024-11-11T16:17:01,137 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/903e7dd5d39144ce90331fe4861f41e4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/903e7dd5d39144ce90331fe4861f41e4 2024-11-11T16:17:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:01,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:01,143 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/B of 70a39dbdbb2217d780a9c88575e9f99d into 903e7dd5d39144ce90331fe4861f41e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:17:01,143 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:01,143 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/B, priority=13, startTime=1731341820713; duration=0sec 2024-11-11T16:17:01,143 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:01,143 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:B 2024-11-11T16:17:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341881162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341881163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341881164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341881166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,257 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/7130495acfa84fc8b0bd108f0b0d9aa0 2024-11-11T16:17:01,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/4622d001f0bc4b0c990a206e4b699fe7 is 50, key is test_row_0/C:col10/1731341818982/Put/seqid=0 2024-11-11T16:17:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341881276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341881276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341881276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341881276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742486_1662 (size=12301) 2024-11-11T16:17:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341881486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341881486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341881486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341881486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,702 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/4622d001f0bc4b0c990a206e4b699fe7 2024-11-11T16:17:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/ec515547879b4264b59a4f264e7681ca as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/ec515547879b4264b59a4f264e7681ca 2024-11-11T16:17:01,710 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/ec515547879b4264b59a4f264e7681ca, entries=150, sequenceid=647, filesize=30.5 K 2024-11-11T16:17:01,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/7130495acfa84fc8b0bd108f0b0d9aa0 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7130495acfa84fc8b0bd108f0b0d9aa0 2024-11-11T16:17:01,714 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7130495acfa84fc8b0bd108f0b0d9aa0, entries=150, sequenceid=647, filesize=12.0 K 2024-11-11T16:17:01,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/4622d001f0bc4b0c990a206e4b699fe7 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4622d001f0bc4b0c990a206e4b699fe7 2024-11-11T16:17:01,718 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4622d001f0bc4b0c990a206e4b699fe7, entries=150, sequenceid=647, filesize=12.0 K 2024-11-11T16:17:01,719 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 70a39dbdbb2217d780a9c88575e9f99d in 926ms, sequenceid=647, compaction requested=false 2024-11-11T16:17:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T16:17:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T16:17:01,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T16:17:01,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2630 sec 2024-11-11T16:17:01,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.2680 sec 2024-11-11T16:17:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(8855): Flush requested on 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:01,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T16:17:01,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:17:01,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:01,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:17:01,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:01,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:17:01,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:01,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116677081de55e46fb8edb85d8f64cf5ca_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341821162/Put/seqid=0 2024-11-11T16:17:01,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742487_1663 (size=12454) 2024-11-11T16:17:01,825 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:01,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341881819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341881821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341881825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341881825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116677081de55e46fb8edb85d8f64cf5ca_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116677081de55e46fb8edb85d8f64cf5ca_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:01,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/34e752abeb1545b4a015ef2c38bef607, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:01,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/34e752abeb1545b4a015ef2c38bef607 is 175, key is test_row_0/A:col10/1731341821162/Put/seqid=0 2024-11-11T16:17:01,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742488_1664 (size=31255) 2024-11-11T16:17:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341881935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341881935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341881935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341881935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341882146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341882146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341882146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341882146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=664, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/34e752abeb1545b4a015ef2c38bef607 2024-11-11T16:17:02,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/66ba8ddc4ccc471bab6f71539441bd98 is 50, key is test_row_0/B:col10/1731341821162/Put/seqid=0 2024-11-11T16:17:02,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742489_1665 (size=12301) 2024-11-11T16:17:02,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341882456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341882456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341882456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341882456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T16:17:02,636 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:17:02,637 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T16:17:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-11T16:17:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T16:17:02,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T16:17:02,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T16:17:02,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T16:17:02,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/66ba8ddc4ccc471bab6f71539441bd98 2024-11-11T16:17:02,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/44bd7549eb024abe8370aad1472877ff is 50, key is test_row_0/C:col10/1731341821162/Put/seqid=0 2024-11-11T16:17:02,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742490_1666 (size=12301) 2024-11-11T16:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T16:17:02,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T16:17:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:02,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:02,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T16:17:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. as already flushing 2024-11-11T16:17:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:02,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T16:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T16:17:02,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1731341882971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731341882972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57944 deadline: 1731341882972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58006 deadline: 1731341882972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 2024-11-11T16:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 , the old value is region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T16:17:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., hostname=16b413a53992,45121,1731341648557, seqNum=5 because the exception is null or not the one we care about 2024-11-11T16:17:03,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/44bd7549eb024abe8370aad1472877ff 2024-11-11T16:17:03,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/34e752abeb1545b4a015ef2c38bef607 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/34e752abeb1545b4a015ef2c38bef607 2024-11-11T16:17:03,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/34e752abeb1545b4a015ef2c38bef607, entries=150, sequenceid=664, filesize=30.5 K 2024-11-11T16:17:03,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/66ba8ddc4ccc471bab6f71539441bd98 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/66ba8ddc4ccc471bab6f71539441bd98 2024-11-11T16:17:03,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/66ba8ddc4ccc471bab6f71539441bd98, entries=150, sequenceid=664, filesize=12.0 K 2024-11-11T16:17:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/44bd7549eb024abe8370aad1472877ff as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/44bd7549eb024abe8370aad1472877ff 2024-11-11T16:17:03,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/44bd7549eb024abe8370aad1472877ff, entries=150, sequenceid=664, filesize=12.0 K 2024-11-11T16:17:03,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 70a39dbdbb2217d780a9c88575e9f99d in 1304ms, sequenceid=664, compaction requested=true 2024-11-11T16:17:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T16:17:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:03,102 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:17:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T16:17:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 70a39dbdbb2217d780a9c88575e9f99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T16:17:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:17:03,102 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:17:03,103 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:17:03,103 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:17:03,103 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/B is initiating minor compaction (all files) 2024-11-11T16:17:03,103 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/A is initiating minor compaction (all files) 2024-11-11T16:17:03,103 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/B in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:03,103 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/A in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:03,103 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8cce832f484c4950a967cde13ad3e008, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/ec515547879b4264b59a4f264e7681ca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/34e752abeb1545b4a015ef2c38bef607] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=92.3 K 2024-11-11T16:17:03,103 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/903e7dd5d39144ce90331fe4861f41e4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7130495acfa84fc8b0bd108f0b0d9aa0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/66ba8ddc4ccc471bab6f71539441bd98] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.7 K 2024-11-11T16:17:03,103 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:03,103 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. files: [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8cce832f484c4950a967cde13ad3e008, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/ec515547879b4264b59a4f264e7681ca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/34e752abeb1545b4a015ef2c38bef607] 2024-11-11T16:17:03,104 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 903e7dd5d39144ce90331fe4861f41e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731341818949 2024-11-11T16:17:03,104 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cce832f484c4950a967cde13ad3e008, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731341818949 2024-11-11T16:17:03,104 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 7130495acfa84fc8b0bd108f0b0d9aa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731341818967 2024-11-11T16:17:03,104 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec515547879b4264b59a4f264e7681ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731341818967 2024-11-11T16:17:03,104 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 66ba8ddc4ccc471bab6f71539441bd98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1731341821152 2024-11-11T16:17:03,104 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34e752abeb1545b4a015ef2c38bef607, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1731341821152 2024-11-11T16:17:03,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45121 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T16:17:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:03,105 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T16:17:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:17:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:17:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:17:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:03,113 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:03,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111155fada4ec0204fd8b2d990ca33407b1e_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341821822/Put/seqid=0 2024-11-11T16:17:03,115 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#B#compaction#562 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:17:03,115 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/26dcaf7655ab40128d165538cb502c11 is 50, key is test_row_0/B:col10/1731341821162/Put/seqid=0 2024-11-11T16:17:03,118 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111fa18cf54e4c145b5b9bf7fd526f49256_70a39dbdbb2217d780a9c88575e9f99d store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:03,120 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111fa18cf54e4c145b5b9bf7fd526f49256_70a39dbdbb2217d780a9c88575e9f99d, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:03,120 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fa18cf54e4c145b5b9bf7fd526f49256_70a39dbdbb2217d780a9c88575e9f99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:03,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742492_1668 (size=12454) 2024-11-11T16:17:03,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742491_1667 (size=13119) 2024-11-11T16:17:03,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742493_1669 (size=4469) 2024-11-11T16:17:03,172 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#A#compaction#563 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:17:03,172 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/25be1e67f44c446e861f09ea906564e9 is 175, key is test_row_0/A:col10/1731341821162/Put/seqid=0 2024-11-11T16:17:03,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742494_1670 (size=32073) 2024-11-11T16:17:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T16:17:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,529 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111155fada4ec0204fd8b2d990ca33407b1e_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155fada4ec0204fd8b2d990ca33407b1e_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:03,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/46fe3493fedd444eae9ad0878439d2d1, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:03,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/46fe3493fedd444eae9ad0878439d2d1 is 175, key is test_row_0/A:col10/1731341821822/Put/seqid=0 2024-11-11T16:17:03,550 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/26dcaf7655ab40128d165538cb502c11 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/26dcaf7655ab40128d165538cb502c11 2024-11-11T16:17:03,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742495_1671 (size=31255) 2024-11-11T16:17:03,551 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=680, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/46fe3493fedd444eae9ad0878439d2d1 2024-11-11T16:17:03,557 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/B of 70a39dbdbb2217d780a9c88575e9f99d into 26dcaf7655ab40128d165538cb502c11(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:17:03,557 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:03,557 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/B, priority=13, startTime=1731341823102; duration=0sec 2024-11-11T16:17:03,557 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T16:17:03,557 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:B 2024-11-11T16:17:03,557 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T16:17:03,558 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T16:17:03,558 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1541): 70a39dbdbb2217d780a9c88575e9f99d/C is initiating minor compaction (all files) 2024-11-11T16:17:03,558 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 70a39dbdbb2217d780a9c88575e9f99d/C in TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:03,558 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c45d7d097eec459880808fa505e179a4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4622d001f0bc4b0c990a206e4b699fe7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/44bd7549eb024abe8370aad1472877ff] into tmpdir=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp, totalSize=36.7 K 2024-11-11T16:17:03,559 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting c45d7d097eec459880808fa505e179a4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731341818949 2024-11-11T16:17:03,559 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 4622d001f0bc4b0c990a206e4b699fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731341818967 2024-11-11T16:17:03,559 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] compactions.Compactor(225): Compacting 44bd7549eb024abe8370aad1472877ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1731341821152 2024-11-11T16:17:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/3c92128a86104cf5b84fcf44128b2e77 is 50, key is test_row_0/B:col10/1731341821822/Put/seqid=0 2024-11-11T16:17:03,571 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 70a39dbdbb2217d780a9c88575e9f99d#C#compaction#565 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T16:17:03,571 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/1d30786edbbd4a55987a84b9d9a73e25 is 50, key is test_row_0/C:col10/1731341821162/Put/seqid=0 2024-11-11T16:17:03,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742496_1672 (size=12301) 2024-11-11T16:17:03,574 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/3c92128a86104cf5b84fcf44128b2e77 2024-11-11T16:17:03,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/30a1818fbd7042d487165fb54574f435 is 50, key is test_row_0/C:col10/1731341821822/Put/seqid=0 2024-11-11T16:17:03,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742497_1673 (size=13119) 2024-11-11T16:17:03,597 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/25be1e67f44c446e861f09ea906564e9 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/25be1e67f44c446e861f09ea906564e9 2024-11-11T16:17:03,602 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/1d30786edbbd4a55987a84b9d9a73e25 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/1d30786edbbd4a55987a84b9d9a73e25 2024-11-11T16:17:03,607 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/A of 70a39dbdbb2217d780a9c88575e9f99d into 25be1e67f44c446e861f09ea906564e9(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:17:03,607 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:03,607 INFO [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/A, priority=13, startTime=1731341823102; duration=0sec 2024-11-11T16:17:03,607 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:03,607 DEBUG [RS:0;16b413a53992:45121-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:A 2024-11-11T16:17:03,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742498_1674 (size=12301) 2024-11-11T16:17:03,609 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/30a1818fbd7042d487165fb54574f435 2024-11-11T16:17:03,610 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 70a39dbdbb2217d780a9c88575e9f99d/C of 70a39dbdbb2217d780a9c88575e9f99d into 1d30786edbbd4a55987a84b9d9a73e25(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T16:17:03,610 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:03,611 INFO [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., storeName=70a39dbdbb2217d780a9c88575e9f99d/C, priority=13, startTime=1731341823102; duration=0sec 2024-11-11T16:17:03,611 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T16:17:03,611 DEBUG [RS:0;16b413a53992:45121-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 70a39dbdbb2217d780a9c88575e9f99d:C 2024-11-11T16:17:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/46fe3493fedd444eae9ad0878439d2d1 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/46fe3493fedd444eae9ad0878439d2d1 2024-11-11T16:17:03,626 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/46fe3493fedd444eae9ad0878439d2d1, entries=150, sequenceid=680, filesize=30.5 K 2024-11-11T16:17:03,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/3c92128a86104cf5b84fcf44128b2e77 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3c92128a86104cf5b84fcf44128b2e77 2024-11-11T16:17:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,632 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3c92128a86104cf5b84fcf44128b2e77, entries=150, sequenceid=680, filesize=12.0 K 2024-11-11T16:17:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/30a1818fbd7042d487165fb54574f435 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/30a1818fbd7042d487165fb54574f435 2024-11-11T16:17:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,638 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/30a1818fbd7042d487165fb54574f435, entries=150, sequenceid=680, filesize=12.0 K 2024-11-11T16:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,639 INFO [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 70a39dbdbb2217d780a9c88575e9f99d in 534ms, sequenceid=680, compaction requested=false 2024-11-11T16:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 70a39dbdbb2217d780a9c88575e9f99d: 2024-11-11T16:17:03,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:03,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/16b413a53992:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-11T16:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-11T16:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T16:17:03,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2024-11-11T16:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0070 sec 2024-11-11T16:17:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,670 INFO [Thread-2544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2544. 2024-11-11T16:17:03,670 DEBUG [Thread-2544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,670 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,670 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,686 INFO [Thread-2540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2540. 2024-11-11T16:17:03,686 DEBUG [Thread-2540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,686 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,686 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,702 INFO [Thread-2536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2536. 2024-11-11T16:17:03,702 DEBUG [Thread-2536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,702 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,702 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,714 INFO [Thread-2542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2542. 2024-11-11T16:17:03,714 DEBUG [Thread-2542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,714 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,714 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45121 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:03,724 INFO [Thread-2538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2538. 2024-11-11T16:17:03,724 DEBUG [Thread-2538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,724 DEBUG [Thread-2538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,724 DEBUG [Thread-2538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T16:17:03,776 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:17:03,988 INFO [Thread-2529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2529. 2024-11-11T16:17:03,988 INFO [Thread-2531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2531. 2024-11-11T16:17:03,988 INFO [Thread-2527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2527. 2024-11-11T16:17:03,988 DEBUG [Thread-2527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,988 DEBUG [Thread-2531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,988 DEBUG [Thread-2529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,988 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,988 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,988 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,988 DEBUG [Thread-2527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,988 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,988 DEBUG [Thread-2529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:03,990 INFO [Thread-2533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2533. 2024-11-11T16:17:03,990 DEBUG [Thread-2533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:03,990 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,990 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:03,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:06,365 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T16:17:09,289 INFO [Thread-2525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2525. 2024-11-11T16:17:09,289 DEBUG [Thread-2525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T16:17:09,289 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:09,289 DEBUG [Thread-2525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:09,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 166 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2281 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2297 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2360 2024-11-11T16:17:09,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2273 2024-11-11T16:17:09,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2288 2024-11-11T16:17:09,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T16:17:09,290 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:17:09,290 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:17:09,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:09,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:09,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:09,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T16:17:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T16:17:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T16:17:09,295 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341829294"}]},"ts":"1731341829294"} 2024-11-11T16:17:09,296 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T16:17:09,296 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T16:17:09,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T16:17:09,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, UNASSIGN}] 2024-11-11T16:17:09,299 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, UNASSIGN 2024-11-11T16:17:09,299 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=CLOSING, regionLocation=16b413a53992,45121,1731341648557 2024-11-11T16:17:09,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, UNASSIGN because future has completed 2024-11-11T16:17:09,301 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T16:17:09,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557}] 2024-11-11T16:17:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T16:17:09,454 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:09,454 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T16:17:09,454 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 70a39dbdbb2217d780a9c88575e9f99d, disabling compactions & flushes 2024-11-11T16:17:09,454 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:09,454 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:09,454 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. after waiting 0 ms 2024-11-11T16:17:09,454 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:09,454 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 70a39dbdbb2217d780a9c88575e9f99d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T16:17:09,454 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=A 2024-11-11T16:17:09,455 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:09,455 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=B 2024-11-11T16:17:09,455 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:09,455 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 70a39dbdbb2217d780a9c88575e9f99d, store=C 2024-11-11T16:17:09,455 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T16:17:09,461 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d8b6cfb109c3476ba0f24ddfce6b85d1_70a39dbdbb2217d780a9c88575e9f99d is 50, key is test_row_0/A:col10/1731341823988/Put/seqid=0 2024-11-11T16:17:09,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742499_1675 (size=12454) 2024-11-11T16:17:09,469 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T16:17:09,483 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d8b6cfb109c3476ba0f24ddfce6b85d1_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d8b6cfb109c3476ba0f24ddfce6b85d1_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:09,485 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/8a77b775a5404449936cc3e5acd379fb, store: [table=TestAcidGuarantees family=A region=70a39dbdbb2217d780a9c88575e9f99d] 2024-11-11T16:17:09,486 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/8a77b775a5404449936cc3e5acd379fb is 175, key is test_row_0/A:col10/1731341823988/Put/seqid=0 2024-11-11T16:17:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742500_1676 (size=31255) 2024-11-11T16:17:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T16:17:09,892 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=691, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/8a77b775a5404449936cc3e5acd379fb 2024-11-11T16:17:09,897 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/0a05c2a53c60408fb4468e07f54a87bf is 50, key is test_row_0/B:col10/1731341823988/Put/seqid=0 2024-11-11T16:17:09,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742501_1677 (size=12301) 2024-11-11T16:17:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T16:17:10,301 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=691 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/0a05c2a53c60408fb4468e07f54a87bf 2024-11-11T16:17:10,309 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/0852ce1ea16141798d28a5d5fbae73af is 50, key is test_row_0/C:col10/1731341823988/Put/seqid=0 2024-11-11T16:17:10,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742502_1678 (size=12301) 2024-11-11T16:17:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T16:17:10,713 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=691 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/0852ce1ea16141798d28a5d5fbae73af 2024-11-11T16:17:10,717 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/A/8a77b775a5404449936cc3e5acd379fb as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8a77b775a5404449936cc3e5acd379fb 2024-11-11T16:17:10,720 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8a77b775a5404449936cc3e5acd379fb, entries=150, sequenceid=691, filesize=30.5 K 2024-11-11T16:17:10,721 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/B/0a05c2a53c60408fb4468e07f54a87bf as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/0a05c2a53c60408fb4468e07f54a87bf 2024-11-11T16:17:10,724 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/0a05c2a53c60408fb4468e07f54a87bf, entries=150, sequenceid=691, filesize=12.0 K 2024-11-11T16:17:10,725 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/.tmp/C/0852ce1ea16141798d28a5d5fbae73af as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0852ce1ea16141798d28a5d5fbae73af 2024-11-11T16:17:10,728 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0852ce1ea16141798d28a5d5fbae73af, entries=150, sequenceid=691, filesize=12.0 K 2024-11-11T16:17:10,729 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 70a39dbdbb2217d780a9c88575e9f99d in 1275ms, sequenceid=691, compaction requested=true 2024-11-11T16:17:10,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/87497efbf56b483bb6290eaedbe0fa5c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/19e2f55a126f40398dd198222bf25f22, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/c84b41d5baeb427fb9c084666b3090f1, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/646783b08fd24b6083539835bb0c4421, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/315f18f681ca42b2b92a56f9d9b89b79, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2706fd1b03aa466b81f2f175de312f98, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/5b0c6e718db6455cb1e3800b679bab45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/d6848ad377f34ad3bb39855c93119f91, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/11104a78e92e480e89e41c4aff1db276, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/e558016fb16f4835825da9f86d4b2aac, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a74f3041bd6c415cbcd328eccd19fc45, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a85db3b56a694a049547beb212e5399f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/18b328d0c1724e00b56a34cf9a2169a3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43736b8b668147eaa97ebe2c36afde23, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/674132344b4a4cd9b4ad8a1803d3b927, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43f1b6a6764741098d48ae67fefbba14, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/3bf653a3fe37496f899c48251fbd6374, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/1479fd91c8304e20b2225b0e476654a3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/9652e5f856474eeb9338e462e143d10c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2e97381461fe4d73b7032651ee984a74, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8cce832f484c4950a967cde13ad3e008, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/94f787725f044aba9dbeca4236791bfe, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/ec515547879b4264b59a4f264e7681ca, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/34e752abeb1545b4a015ef2c38bef607] to archive 2024-11-11T16:17:10,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:17:10,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/87497efbf56b483bb6290eaedbe0fa5c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/87497efbf56b483bb6290eaedbe0fa5c 2024-11-11T16:17:10,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/19e2f55a126f40398dd198222bf25f22 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/19e2f55a126f40398dd198222bf25f22 2024-11-11T16:17:10,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/c84b41d5baeb427fb9c084666b3090f1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/c84b41d5baeb427fb9c084666b3090f1 2024-11-11T16:17:10,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/646783b08fd24b6083539835bb0c4421 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/646783b08fd24b6083539835bb0c4421 2024-11-11T16:17:10,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/315f18f681ca42b2b92a56f9d9b89b79 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/315f18f681ca42b2b92a56f9d9b89b79 2024-11-11T16:17:10,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2706fd1b03aa466b81f2f175de312f98 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2706fd1b03aa466b81f2f175de312f98 2024-11-11T16:17:10,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/5b0c6e718db6455cb1e3800b679bab45 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/5b0c6e718db6455cb1e3800b679bab45 2024-11-11T16:17:10,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/d6848ad377f34ad3bb39855c93119f91 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/d6848ad377f34ad3bb39855c93119f91 2024-11-11T16:17:10,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/11104a78e92e480e89e41c4aff1db276 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/11104a78e92e480e89e41c4aff1db276 2024-11-11T16:17:10,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/e558016fb16f4835825da9f86d4b2aac to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/e558016fb16f4835825da9f86d4b2aac 2024-11-11T16:17:10,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a74f3041bd6c415cbcd328eccd19fc45 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a74f3041bd6c415cbcd328eccd19fc45 2024-11-11T16:17:10,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a85db3b56a694a049547beb212e5399f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/a85db3b56a694a049547beb212e5399f 2024-11-11T16:17:10,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/18b328d0c1724e00b56a34cf9a2169a3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/18b328d0c1724e00b56a34cf9a2169a3 2024-11-11T16:17:10,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43736b8b668147eaa97ebe2c36afde23 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43736b8b668147eaa97ebe2c36afde23 2024-11-11T16:17:10,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/674132344b4a4cd9b4ad8a1803d3b927 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/674132344b4a4cd9b4ad8a1803d3b927 2024-11-11T16:17:10,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43f1b6a6764741098d48ae67fefbba14 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/43f1b6a6764741098d48ae67fefbba14 2024-11-11T16:17:10,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/3bf653a3fe37496f899c48251fbd6374 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/3bf653a3fe37496f899c48251fbd6374 2024-11-11T16:17:10,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/1479fd91c8304e20b2225b0e476654a3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/1479fd91c8304e20b2225b0e476654a3 2024-11-11T16:17:10,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/9652e5f856474eeb9338e462e143d10c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/9652e5f856474eeb9338e462e143d10c 2024-11-11T16:17:10,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2e97381461fe4d73b7032651ee984a74 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/2e97381461fe4d73b7032651ee984a74 2024-11-11T16:17:10,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8cce832f484c4950a967cde13ad3e008 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8cce832f484c4950a967cde13ad3e008 2024-11-11T16:17:10,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/94f787725f044aba9dbeca4236791bfe to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/94f787725f044aba9dbeca4236791bfe 2024-11-11T16:17:10,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/ec515547879b4264b59a4f264e7681ca to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/ec515547879b4264b59a4f264e7681ca 2024-11-11T16:17:10,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/34e752abeb1545b4a015ef2c38bef607 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/34e752abeb1545b4a015ef2c38bef607 2024-11-11T16:17:10,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/188dbd102b494c77b5676aad16797434, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/47926eb9e6c44685916fb0642ba57044, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/bd742529544d4f5fadb80d9c4ab486bc, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/ec8fcac55a6049ccadcb3333924b0386, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/92d4baf730c240f5af7789e9dacd5f9a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/50be2990dd864b778e158a1095a35a70, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/6e12905d826f40b0861ae194357f00bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/708d53b81895491e987f4276eb6abdde, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7bd4b3e11bf54cb89aa22039abdbc7c4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4008049363c84e9d81f9dfe8b761c50e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/e94afad066fb4c22bb33bce9bee9897d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/96b2a28b75ea499eb2d1015d639b13bb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/192ddf8efdca40a1973ec9f3c8622b6e, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/acbc9bff2008484ab3afde7e4deea01f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/c08e45c2aaa74ac5927033529e61fc8d, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/b7b5a6a1ab494989acdfeb8b8e5c2922, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4a5848cbf6c54c528fa7ac8f17a04b4a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3b04732c32644c87bb123562f9c0351f, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/38c9e756733946159acdb218c6e2a720, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/a2bf42ad06c943f18d5943cae26f7bf5, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/903e7dd5d39144ce90331fe4861f41e4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/d80a72a0f9634c0e83f48379ee046036, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7130495acfa84fc8b0bd108f0b0d9aa0, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/66ba8ddc4ccc471bab6f71539441bd98] to archive 2024-11-11T16:17:10,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:17:10,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/188dbd102b494c77b5676aad16797434 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/188dbd102b494c77b5676aad16797434 2024-11-11T16:17:10,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/47926eb9e6c44685916fb0642ba57044 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/47926eb9e6c44685916fb0642ba57044 2024-11-11T16:17:10,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/bd742529544d4f5fadb80d9c4ab486bc to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/bd742529544d4f5fadb80d9c4ab486bc 2024-11-11T16:17:10,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/ec8fcac55a6049ccadcb3333924b0386 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/ec8fcac55a6049ccadcb3333924b0386 2024-11-11T16:17:10,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/92d4baf730c240f5af7789e9dacd5f9a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/92d4baf730c240f5af7789e9dacd5f9a 2024-11-11T16:17:10,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/50be2990dd864b778e158a1095a35a70 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/50be2990dd864b778e158a1095a35a70 2024-11-11T16:17:10,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/6e12905d826f40b0861ae194357f00bb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/6e12905d826f40b0861ae194357f00bb 2024-11-11T16:17:10,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/708d53b81895491e987f4276eb6abdde to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/708d53b81895491e987f4276eb6abdde 2024-11-11T16:17:10,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7bd4b3e11bf54cb89aa22039abdbc7c4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7bd4b3e11bf54cb89aa22039abdbc7c4 2024-11-11T16:17:10,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4008049363c84e9d81f9dfe8b761c50e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4008049363c84e9d81f9dfe8b761c50e 2024-11-11T16:17:10,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/e94afad066fb4c22bb33bce9bee9897d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/e94afad066fb4c22bb33bce9bee9897d 2024-11-11T16:17:10,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/96b2a28b75ea499eb2d1015d639b13bb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/96b2a28b75ea499eb2d1015d639b13bb 2024-11-11T16:17:10,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/192ddf8efdca40a1973ec9f3c8622b6e to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/192ddf8efdca40a1973ec9f3c8622b6e 2024-11-11T16:17:10,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/acbc9bff2008484ab3afde7e4deea01f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/acbc9bff2008484ab3afde7e4deea01f 2024-11-11T16:17:10,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/c08e45c2aaa74ac5927033529e61fc8d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/c08e45c2aaa74ac5927033529e61fc8d 2024-11-11T16:17:10,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/b7b5a6a1ab494989acdfeb8b8e5c2922 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/b7b5a6a1ab494989acdfeb8b8e5c2922 2024-11-11T16:17:10,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4a5848cbf6c54c528fa7ac8f17a04b4a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/4a5848cbf6c54c528fa7ac8f17a04b4a 2024-11-11T16:17:10,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3b04732c32644c87bb123562f9c0351f to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3b04732c32644c87bb123562f9c0351f 2024-11-11T16:17:10,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/38c9e756733946159acdb218c6e2a720 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/38c9e756733946159acdb218c6e2a720 2024-11-11T16:17:10,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/a2bf42ad06c943f18d5943cae26f7bf5 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/a2bf42ad06c943f18d5943cae26f7bf5 2024-11-11T16:17:10,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/903e7dd5d39144ce90331fe4861f41e4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/903e7dd5d39144ce90331fe4861f41e4 2024-11-11T16:17:10,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/d80a72a0f9634c0e83f48379ee046036 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/d80a72a0f9634c0e83f48379ee046036 2024-11-11T16:17:10,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7130495acfa84fc8b0bd108f0b0d9aa0 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/7130495acfa84fc8b0bd108f0b0d9aa0 2024-11-11T16:17:10,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/66ba8ddc4ccc471bab6f71539441bd98 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/66ba8ddc4ccc471bab6f71539441bd98 2024-11-11T16:17:10,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/ca6b5d7873134992883a78b5dd3e7d78, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0c1be218e6324fb1a9862cfd5d668820, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7ed3a8f7961d4b16a861bcad2e6ec0fb, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/b78cf19e32384423923b1b6aec7fa3a6, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/2cbe4077b6c5432dbe7c23d38312485c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/05c8d49996954ce9b3a70ae3326a16ba, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af4d7df3e6cc4a0caeebfe638957247b, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/a1614495616b406f8ffbeb8986d198b8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/9a302b0dba1741b2acd5f90088585744, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6c109f1be89645b0bbe25ec6a69e38f8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cb4e9b9acc5045e59631b0d9e8fe0186, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/df782852f2414d378584598d116f79b7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c191cf7e8f414aa69eba8548a5ed7664, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/f9af84299ce44772b8a7efbe71b7fac3, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6a2b823fffe4462d988f9b5ac552e2c8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cfd0090fe7434bfdb792fd037e721e8a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/03babbe272544a5b8a65ede5ce13fe2a, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af5e8784063844679fd5f77a92cc0293, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4586b4b62ff749ddba1fd23c745cd327, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/518f73011ec042e089a8a7dd3c70e91c, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c45d7d097eec459880808fa505e179a4, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7753812b20fa4d89a940dd3c935a96e8, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4622d001f0bc4b0c990a206e4b699fe7, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/44bd7549eb024abe8370aad1472877ff] to archive 2024-11-11T16:17:10,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T16:17:10,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/ca6b5d7873134992883a78b5dd3e7d78 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/ca6b5d7873134992883a78b5dd3e7d78 2024-11-11T16:17:10,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0c1be218e6324fb1a9862cfd5d668820 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0c1be218e6324fb1a9862cfd5d668820 2024-11-11T16:17:10,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7ed3a8f7961d4b16a861bcad2e6ec0fb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7ed3a8f7961d4b16a861bcad2e6ec0fb 2024-11-11T16:17:10,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/b78cf19e32384423923b1b6aec7fa3a6 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/b78cf19e32384423923b1b6aec7fa3a6 2024-11-11T16:17:10,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/2cbe4077b6c5432dbe7c23d38312485c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/2cbe4077b6c5432dbe7c23d38312485c 2024-11-11T16:17:10,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/05c8d49996954ce9b3a70ae3326a16ba to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/05c8d49996954ce9b3a70ae3326a16ba 2024-11-11T16:17:10,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af4d7df3e6cc4a0caeebfe638957247b to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af4d7df3e6cc4a0caeebfe638957247b 2024-11-11T16:17:10,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/a1614495616b406f8ffbeb8986d198b8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/a1614495616b406f8ffbeb8986d198b8 2024-11-11T16:17:10,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/9a302b0dba1741b2acd5f90088585744 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/9a302b0dba1741b2acd5f90088585744 2024-11-11T16:17:10,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6c109f1be89645b0bbe25ec6a69e38f8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6c109f1be89645b0bbe25ec6a69e38f8 2024-11-11T16:17:10,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cb4e9b9acc5045e59631b0d9e8fe0186 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cb4e9b9acc5045e59631b0d9e8fe0186 2024-11-11T16:17:10,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/df782852f2414d378584598d116f79b7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/df782852f2414d378584598d116f79b7 2024-11-11T16:17:10,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c191cf7e8f414aa69eba8548a5ed7664 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c191cf7e8f414aa69eba8548a5ed7664 2024-11-11T16:17:10,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/f9af84299ce44772b8a7efbe71b7fac3 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/f9af84299ce44772b8a7efbe71b7fac3 2024-11-11T16:17:10,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6a2b823fffe4462d988f9b5ac552e2c8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/6a2b823fffe4462d988f9b5ac552e2c8 2024-11-11T16:17:10,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cfd0090fe7434bfdb792fd037e721e8a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/cfd0090fe7434bfdb792fd037e721e8a 2024-11-11T16:17:10,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/03babbe272544a5b8a65ede5ce13fe2a to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/03babbe272544a5b8a65ede5ce13fe2a 2024-11-11T16:17:10,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af5e8784063844679fd5f77a92cc0293 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/af5e8784063844679fd5f77a92cc0293 2024-11-11T16:17:10,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4586b4b62ff749ddba1fd23c745cd327 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4586b4b62ff749ddba1fd23c745cd327 2024-11-11T16:17:10,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/518f73011ec042e089a8a7dd3c70e91c to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/518f73011ec042e089a8a7dd3c70e91c 2024-11-11T16:17:10,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c45d7d097eec459880808fa505e179a4 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/c45d7d097eec459880808fa505e179a4 2024-11-11T16:17:10,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7753812b20fa4d89a940dd3c935a96e8 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/7753812b20fa4d89a940dd3c935a96e8 2024-11-11T16:17:10,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4622d001f0bc4b0c990a206e4b699fe7 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/4622d001f0bc4b0c990a206e4b699fe7 2024-11-11T16:17:10,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/44bd7549eb024abe8370aad1472877ff to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/44bd7549eb024abe8370aad1472877ff 2024-11-11T16:17:10,824 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/recovered.edits/694.seqid, newMaxSeqId=694, maxSeqId=4 2024-11-11T16:17:10,825 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. 2024-11-11T16:17:10,825 DEBUG [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 70a39dbdbb2217d780a9c88575e9f99d: Waiting for close lock at 1731341829454Running coprocessor pre-close hooks at 1731341829454Disabling compacts and flushes for region at 1731341829454Disabling writes for close at 1731341829454Obtaining lock to block concurrent updates at 1731341829454Preparing flush snapshotting stores in 70a39dbdbb2217d780a9c88575e9f99d at 1731341829454Finished memstore snapshotting TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731341829455 (+1 ms)Flushing stores of TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d. at 1731341829455Flushing 70a39dbdbb2217d780a9c88575e9f99d/A: creating writer at 1731341829455Flushing mob file 70a39dbdbb2217d780a9c88575e9f99d/A: appending metadata at 1731341829461 (+6 ms)Flushing mob file 70a39dbdbb2217d780a9c88575e9f99d/A: closing flushed file at 1731341829461Flushing 70a39dbdbb2217d780a9c88575e9f99d/A: appending metadata at 1731341829485 (+24 ms)Flushing 70a39dbdbb2217d780a9c88575e9f99d/A: closing flushed file at 1731341829485Flushing 70a39dbdbb2217d780a9c88575e9f99d/B: creating writer at 1731341829895 (+410 ms)Flushing 70a39dbdbb2217d780a9c88575e9f99d/B: appending metadata at 1731341829897 (+2 ms)Flushing 70a39dbdbb2217d780a9c88575e9f99d/B: closing flushed file at 1731341829897Flushing 70a39dbdbb2217d780a9c88575e9f99d/C: creating writer at 1731341830306 (+409 ms)Flushing 70a39dbdbb2217d780a9c88575e9f99d/C: appending metadata at 1731341830308 (+2 ms)Flushing 70a39dbdbb2217d780a9c88575e9f99d/C: closing flushed file at 1731341830308Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73270892: reopening flushed file at 1731341830716 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61f85cc7: reopening flushed file at 1731341830720 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5751ed21: reopening flushed file at 1731341830724 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 70a39dbdbb2217d780a9c88575e9f99d in 1275ms, sequenceid=691, compaction requested=true at 1731341830729 (+5 ms)Writing region close event to WAL at 1731341830821 (+92 ms)Running coprocessor post-close hooks at 1731341830825 (+4 ms)Closed at 1731341830825 2024-11-11T16:17:10,827 INFO [RS_CLOSE_REGION-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:10,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=70a39dbdbb2217d780a9c88575e9f99d, regionState=CLOSED 2024-11-11T16:17:10,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 because future has completed 2024-11-11T16:17:10,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T16:17:10,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 70a39dbdbb2217d780a9c88575e9f99d, server=16b413a53992,45121,1731341648557 in 1.5280 sec 2024-11-11T16:17:10,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-11T16:17:10,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=70a39dbdbb2217d780a9c88575e9f99d, UNASSIGN in 1.5330 sec 2024-11-11T16:17:10,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T16:17:10,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-11T16:17:10,837 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731341830836"}]},"ts":"1731341830836"} 2024-11-11T16:17:10,838 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T16:17:10,838 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T16:17:10,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5480 sec 2024-11-11T16:17:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T16:17:11,437 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:17:11,437 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T16:17:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:17:11,440 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:17:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T16:17:11,441 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:17:11,443 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,445 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C, FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/recovered.edits] 2024-11-11T16:17:11,450 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/25be1e67f44c446e861f09ea906564e9 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/25be1e67f44c446e861f09ea906564e9 2024-11-11T16:17:11,451 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/46fe3493fedd444eae9ad0878439d2d1 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/46fe3493fedd444eae9ad0878439d2d1 2024-11-11T16:17:11,453 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8a77b775a5404449936cc3e5acd379fb to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/A/8a77b775a5404449936cc3e5acd379fb 2024-11-11T16:17:11,455 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/0a05c2a53c60408fb4468e07f54a87bf to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/0a05c2a53c60408fb4468e07f54a87bf 2024-11-11T16:17:11,456 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/26dcaf7655ab40128d165538cb502c11 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/26dcaf7655ab40128d165538cb502c11 2024-11-11T16:17:11,457 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3c92128a86104cf5b84fcf44128b2e77 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/B/3c92128a86104cf5b84fcf44128b2e77 2024-11-11T16:17:11,459 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0852ce1ea16141798d28a5d5fbae73af to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/0852ce1ea16141798d28a5d5fbae73af 2024-11-11T16:17:11,460 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/1d30786edbbd4a55987a84b9d9a73e25 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/1d30786edbbd4a55987a84b9d9a73e25 2024-11-11T16:17:11,462 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/30a1818fbd7042d487165fb54574f435 to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/C/30a1818fbd7042d487165fb54574f435 2024-11-11T16:17:11,466 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/recovered.edits/694.seqid to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d/recovered.edits/694.seqid 2024-11-11T16:17:11,466 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/default/TestAcidGuarantees/70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,466 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T16:17:11,467 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T16:17:11,467 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T16:17:11,470 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102c52acf820341ca9bc9928f632bc4f8_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102c52acf820341ca9bc9928f632bc4f8_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,471 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111107bee693c8194e088d9ddf76eee3eee4_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111107bee693c8194e088d9ddf76eee3eee4_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,472 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110d5fd29aa29c434f9ece5bd0d4f64a55_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110d5fd29aa29c434f9ece5bd0d4f64a55_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,474 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110509f5255f4410db312a06ffcf8359e_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110509f5255f4410db312a06ffcf8359e_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,475 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111814ceb3e2e54f088a73fb865fbd2b4e_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111814ceb3e2e54f088a73fb865fbd2b4e_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,476 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111205d8845e0fb4cf5a3645043563bfb28_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111205d8845e0fb4cf5a3645043563bfb28_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,477 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115007fb80eaec47d0925702893e3202a3_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115007fb80eaec47d0925702893e3202a3_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,478 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155fada4ec0204fd8b2d990ca33407b1e_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155fada4ec0204fd8b2d990ca33407b1e_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,480 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111575dcb4275b74fa3ac7ffa3affb2035a_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111575dcb4275b74fa3ac7ffa3affb2035a_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,481 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e66a281cba04d7cb459445b53aaa21b_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e66a281cba04d7cb459445b53aaa21b_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,482 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116677081de55e46fb8edb85d8f64cf5ca_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116677081de55e46fb8edb85d8f64cf5ca_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,484 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116c7fb3cae7ec47dfb50f539898b59068_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116c7fb3cae7ec47dfb50f539898b59068_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,485 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173d198777eec4a5bb7d33cdd82c6a506_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173d198777eec4a5bb7d33cdd82c6a506_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,486 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111787de8103a854ba79495a2e40cf78713_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111787de8103a854ba79495a2e40cf78713_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,488 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118397ccb56cda4dae8d0940c7ce36b3d1_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118397ccb56cda4dae8d0940c7ce36b3d1_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,489 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193f07958c3e54d278fb980b6c2924911_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193f07958c3e54d278fb980b6c2924911_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,490 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b586b536a9fc4a4ebaba31841bab44e8_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b586b536a9fc4a4ebaba31841bab44e8_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,492 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c1d7583525914f25bff0c7a9e14fbbf1_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c1d7583525914f25bff0c7a9e14fbbf1_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,493 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d8b6cfb109c3476ba0f24ddfce6b85d1_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d8b6cfb109c3476ba0f24ddfce6b85d1_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,494 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ed76d58a69604edc81fd1a8948846116_70a39dbdbb2217d780a9c88575e9f99d to hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ed76d58a69604edc81fd1a8948846116_70a39dbdbb2217d780a9c88575e9f99d 2024-11-11T16:17:11,495 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T16:17:11,497 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:17:11,500 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T16:17:11,503 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T16:17:11,504 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:17:11,504 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T16:17:11,504 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731341831504"}]},"ts":"9223372036854775807"} 2024-11-11T16:17:11,506 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T16:17:11,507 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 70a39dbdbb2217d780a9c88575e9f99d, NAME => 'TestAcidGuarantees,,1731341800558.70a39dbdbb2217d780a9c88575e9f99d.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T16:17:11,507 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T16:17:11,507 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731341831507"}]},"ts":"9223372036854775807"} 2024-11-11T16:17:11,508 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T16:17:11,509 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T16:17:11,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-11-11T16:17:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T16:17:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T16:17:11,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T16:17:11,558 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=605 (was 532) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1023 (was 1890) 2024-11-11T16:17:11,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T16:17:11,559 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:17:11,559 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:17:11,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:11,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:11,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T16:17:11,559 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T16:17:11,559 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=350595032, stopped=false 2024-11-11T16:17:11,560 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=16b413a53992,33601,1731341647570 2024-11-11T16:17:11,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T16:17:11,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T16:17:11,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:17:11,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:17:11,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:17:11,562 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T16:17:11,562 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:17:11,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:11,563 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b413a53992,45121,1731341648557' ***** 2024-11-11T16:17:11,563 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T16:17:11,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:17:11,563 INFO [RS:0;16b413a53992:45121 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T16:17:11,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(959): stopping server 16b413a53992,45121,1731341648557 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T16:17:11,564 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;16b413a53992:45121. 2024-11-11T16:17:11,564 DEBUG [RS:0;16b413a53992:45121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T16:17:11,564 DEBUG [RS:0;16b413a53992:45121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T16:17:11,564 INFO [RS:0;16b413a53992:45121 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T16:17:11,565 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T16:17:11,565 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T16:17:11,565 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T16:17:11,566 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:11,566 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T16:17:11,566 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T16:17:11,566 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T16:17:11,566 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T16:17:11,566 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T16:17:11,566 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T16:17:11,596 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/info/4201717e3c6c4c848da2827aac263ff5 is 95, key is TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1./info:/1731341684559/DeleteFamily/seqid=0 2024-11-11T16:17:11,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742503_1679 (size=6514) 2024-11-11T16:17:11,657 INFO [regionserver/16b413a53992:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:17:11,766 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:11,966 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:12,000 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/info/4201717e3c6c4c848da2827aac263ff5 2024-11-11T16:17:12,021 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/ns/e2906d9554de45c5947743fd0a285184 is 93, key is TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1./ns:/1731341684508/DeleteFamily/seqid=0 2024-11-11T16:17:12,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742504_1680 (size=5906) 2024-11-11T16:17:12,166 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:12,175 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T16:17:12,175 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-11T16:17:12,175 INFO [regionserver/16b413a53992:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T16:17:12,366 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:12,428 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/ns/e2906d9554de45c5947743fd0a285184 2024-11-11T16:17:12,456 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/rep_barrier/fccbf721ec52451db353e64002305851 is 102, key is TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1./rep_barrier:/1731341684508/DeleteFamily/seqid=0 2024-11-11T16:17:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742505_1681 (size=6025) 2024-11-11T16:17:12,567 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:12,767 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:12,865 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/rep_barrier/fccbf721ec52451db353e64002305851 2024-11-11T16:17:12,882 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/table/028be264e27745279ad722f3631bbf95 is 96, key is TestAcidGuarantees,,1731341653568.af9dedd7afe35ed4a9eb53f0210465b1./table:/1731341684508/DeleteFamily/seqid=0 2024-11-11T16:17:12,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742506_1682 (size=5936) 2024-11-11T16:17:12,967 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:13,167 DEBUG [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T16:17:13,286 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/table/028be264e27745279ad722f3631bbf95 2024-11-11T16:17:13,290 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/info/4201717e3c6c4c848da2827aac263ff5 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/info/4201717e3c6c4c848da2827aac263ff5 2024-11-11T16:17:13,293 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/info/4201717e3c6c4c848da2827aac263ff5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T16:17:13,294 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/ns/e2906d9554de45c5947743fd0a285184 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/ns/e2906d9554de45c5947743fd0a285184 2024-11-11T16:17:13,298 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/ns/e2906d9554de45c5947743fd0a285184, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T16:17:13,298 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/rep_barrier/fccbf721ec52451db353e64002305851 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/rep_barrier/fccbf721ec52451db353e64002305851 2024-11-11T16:17:13,302 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/rep_barrier/fccbf721ec52451db353e64002305851, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T16:17:13,302 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/.tmp/table/028be264e27745279ad722f3631bbf95 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/table/028be264e27745279ad722f3631bbf95 2024-11-11T16:17:13,305 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/table/028be264e27745279ad722f3631bbf95, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T16:17:13,306 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-11T16:17:13,315 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T16:17:13,316 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T16:17:13,316 INFO [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T16:17:13,316 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731341831565Running coprocessor pre-close hooks at 1731341831566 (+1 ms)Disabling compacts and flushes for region at 1731341831566Disabling writes for close at 1731341831566Obtaining lock to block concurrent updates at 1731341831566Preparing flush snapshotting stores in 1588230740 at 1731341831566Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731341831567 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731341831568 (+1 ms)Flushing 1588230740/info: creating writer at 1731341831568Flushing 1588230740/info: appending metadata at 1731341831595 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731341831595Flushing 1588230740/ns: creating writer at 1731341832003 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731341832021 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731341832021Flushing 1588230740/rep_barrier: creating writer at 1731341832435 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731341832455 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731341832455Flushing 1588230740/table: creating writer at 1731341832868 (+413 ms)Flushing 1588230740/table: appending metadata at 1731341832882 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731341832882Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c4edb54: reopening flushed file at 1731341833289 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4699cee6: reopening flushed file at 1731341833294 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fef4c16: reopening flushed file at 1731341833298 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cdfbb58: reopening flushed file at 1731341833302 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1731341833306 (+4 ms)Writing region close event to WAL at 1731341833309 (+3 ms)Running coprocessor post-close hooks at 1731341833316 (+7 ms)Closed at 1731341833316 2024-11-11T16:17:13,316 DEBUG [RS_CLOSE_META-regionserver/16b413a53992:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T16:17:13,368 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(976): stopping server 16b413a53992,45121,1731341648557; all regions closed. 2024-11-11T16:17:13,376 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/WALs/16b413a53992,45121,1731341648557/16b413a53992%2C45121%2C1731341648557.meta.1731341652857.meta not finished, retry = 0 2024-11-11T16:17:13,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741834_1010 (size=25344) 2024-11-11T16:17:13,479 DEBUG [RS:0;16b413a53992:45121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/oldWALs 2024-11-11T16:17:13,479 INFO [RS:0;16b413a53992:45121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b413a53992%2C45121%2C1731341648557.meta:.meta(num 1731341652857) 2024-11-11T16:17:13,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741833_1009 (size=23310609) 2024-11-11T16:17:13,484 DEBUG [RS:0;16b413a53992:45121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/oldWALs 2024-11-11T16:17:13,484 INFO [RS:0;16b413a53992:45121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b413a53992%2C45121%2C1731341648557:(num 1731341652312) 2024-11-11T16:17:13,484 DEBUG [RS:0;16b413a53992:45121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T16:17:13,484 INFO [RS:0;16b413a53992:45121 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T16:17:13,484 INFO [RS:0;16b413a53992:45121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T16:17:13,484 INFO [RS:0;16b413a53992:45121 {}] hbase.ChoreService(370): Chore service for: regionserver/16b413a53992:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T16:17:13,485 INFO [RS:0;16b413a53992:45121 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T16:17:13,485 INFO [regionserver/16b413a53992:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T16:17:13,487 INFO [RS:0;16b413a53992:45121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45121 2024-11-11T16:17:13,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T16:17:13,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b413a53992,45121,1731341648557 2024-11-11T16:17:13,493 INFO [RS:0;16b413a53992:45121 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T16:17:13,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b413a53992,45121,1731341648557] 2024-11-11T16:17:13,495 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b413a53992,45121,1731341648557 already deleted, retry=false 2024-11-11T16:17:13,496 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16b413a53992,45121,1731341648557 expired; onlineServers=0 2024-11-11T16:17:13,496 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '16b413a53992,33601,1731341647570' ***** 2024-11-11T16:17:13,496 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T16:17:13,496 INFO [M:0;16b413a53992:33601 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T16:17:13,496 INFO [M:0;16b413a53992:33601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T16:17:13,496 DEBUG [M:0;16b413a53992:33601 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T16:17:13,496 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T16:17:13,496 DEBUG [M:0;16b413a53992:33601 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T16:17:13,496 DEBUG [master/16b413a53992:0:becomeActiveMaster-HFileCleaner.large.0-1731341651852 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.large.0-1731341651852,5,FailOnTimeoutGroup] 2024-11-11T16:17:13,496 DEBUG [master/16b413a53992:0:becomeActiveMaster-HFileCleaner.small.0-1731341651853 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b413a53992:0:becomeActiveMaster-HFileCleaner.small.0-1731341651853,5,FailOnTimeoutGroup] 2024-11-11T16:17:13,497 INFO [M:0;16b413a53992:33601 {}] hbase.ChoreService(370): Chore service for: master/16b413a53992:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T16:17:13,497 INFO [M:0;16b413a53992:33601 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T16:17:13,497 DEBUG [M:0;16b413a53992:33601 {}] master.HMaster(1795): Stopping service threads 2024-11-11T16:17:13,497 INFO [M:0;16b413a53992:33601 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T16:17:13,497 INFO [M:0;16b413a53992:33601 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T16:17:13,497 ERROR [M:0;16b413a53992:33601 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T16:17:13,498 INFO [M:0;16b413a53992:33601 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T16:17:13,498 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T16:17:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T16:17:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T16:17:13,500 DEBUG [M:0;16b413a53992:33601 {}] zookeeper.ZKUtil(347): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T16:17:13,500 WARN [M:0;16b413a53992:33601 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T16:17:13,501 INFO [M:0;16b413a53992:33601 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/.lastflushedseqids 2024-11-11T16:17:13,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742507_1683 (size=72) 2024-11-11T16:17:13,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T16:17:13,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45121-0x1002f9f05730001, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T16:17:13,599 INFO [RS:0;16b413a53992:45121 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T16:17:13,599 INFO [RS:0;16b413a53992:45121 {}] regionserver.HRegionServer(1031): Exiting; stopping=16b413a53992,45121,1731341648557; zookeeper connection closed. 2024-11-11T16:17:13,599 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b3f498 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b3f498 2024-11-11T16:17:13,600 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T16:17:13,914 INFO [M:0;16b413a53992:33601 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T16:17:13,914 INFO [M:0;16b413a53992:33601 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T16:17:13,915 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T16:17:13,915 INFO [M:0;16b413a53992:33601 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:17:13,915 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:17:13,915 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T16:17:13,915 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:17:13,915 INFO [M:0;16b413a53992:33601 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=802.74 KB heapSize=985.30 KB 2024-11-11T16:17:13,934 DEBUG [M:0;16b413a53992:33601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a50a3fe201341f9bc0bbbe33420a735 is 82, key is hbase:meta,,1/info:regioninfo/1731341652946/Put/seqid=0 2024-11-11T16:17:13,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742508_1684 (size=5672) 2024-11-11T16:17:14,339 INFO [M:0;16b413a53992:33601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2271 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a50a3fe201341f9bc0bbbe33420a735 2024-11-11T16:17:14,364 DEBUG [M:0;16b413a53992:33601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b00d60ddebcd4cd39dac2b15acfdd7e4 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1731341803634/Put/seqid=0 2024-11-11T16:17:14,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742509_1685 (size=44034) 2024-11-11T16:17:14,769 INFO [M:0;16b413a53992:33601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=802.18 KB at sequenceid=2271 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b00d60ddebcd4cd39dac2b15acfdd7e4 2024-11-11T16:17:14,773 INFO [M:0;16b413a53992:33601 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b00d60ddebcd4cd39dac2b15acfdd7e4 2024-11-11T16:17:14,798 DEBUG [M:0;16b413a53992:33601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84111ac95617465a9618fd21056e33f2 is 69, key is 16b413a53992,45121,1731341648557/rs:state/1731341651970/Put/seqid=0 2024-11-11T16:17:14,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073742510_1686 (size=5156) 2024-11-11T16:17:15,206 INFO [M:0;16b413a53992:33601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2271 (bloomFilter=true), to=hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84111ac95617465a9618fd21056e33f2 2024-11-11T16:17:15,211 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a50a3fe201341f9bc0bbbe33420a735 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5a50a3fe201341f9bc0bbbe33420a735 2024-11-11T16:17:15,215 INFO [M:0;16b413a53992:33601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5a50a3fe201341f9bc0bbbe33420a735, entries=8, sequenceid=2271, filesize=5.5 K 2024-11-11T16:17:15,216 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b00d60ddebcd4cd39dac2b15acfdd7e4 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b00d60ddebcd4cd39dac2b15acfdd7e4 2024-11-11T16:17:15,222 INFO [M:0;16b413a53992:33601 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b00d60ddebcd4cd39dac2b15acfdd7e4 2024-11-11T16:17:15,222 INFO [M:0;16b413a53992:33601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b00d60ddebcd4cd39dac2b15acfdd7e4, entries=162, sequenceid=2271, filesize=43.0 K 2024-11-11T16:17:15,223 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84111ac95617465a9618fd21056e33f2 as hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84111ac95617465a9618fd21056e33f2 2024-11-11T16:17:15,226 INFO [M:0;16b413a53992:33601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33383/user/jenkins/test-data/f3cbf53f-b52c-85a9-291a-be93fdb77669/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84111ac95617465a9618fd21056e33f2, entries=1, sequenceid=2271, filesize=5.0 K 2024-11-11T16:17:15,227 INFO [M:0;16b413a53992:33601 {}] regionserver.HRegion(3140): Finished flush of dataSize ~802.74 KB/822002, heapSize ~985 KB/1008640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=2271, compaction requested=false 2024-11-11T16:17:15,229 INFO [M:0;16b413a53992:33601 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T16:17:15,229 DEBUG [M:0;16b413a53992:33601 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731341833915Disabling compacts and flushes for region at 1731341833915Disabling writes for close at 1731341833915Obtaining lock to block concurrent updates at 1731341833915Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731341833915Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=822002, getHeapSize=1008880, getOffHeapSize=0, getCellsCount=2458 at 1731341833915Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731341833916 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731341833916Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731341833934 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731341833934Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731341834343 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731341834364 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731341834364Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731341834773 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731341834797 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731341834797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a37227c: reopening flushed file at 1731341835210 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@181627bb: reopening flushed file at 1731341835215 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7305b5c: reopening flushed file at 1731341835222 (+7 ms)Finished flush of dataSize ~802.74 KB/822002, heapSize ~985 KB/1008640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=2271, compaction requested=false at 1731341835227 (+5 ms)Writing region close event to WAL at 1731341835229 (+2 ms)Closed at 1731341835229 2024-11-11T16:17:15,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34773 is added to blk_1073741830_1006 (size=970784) 2024-11-11T16:17:15,232 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T16:17:15,232 INFO [M:0;16b413a53992:33601 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T16:17:15,233 INFO [M:0;16b413a53992:33601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33601 2024-11-11T16:17:15,234 INFO [M:0;16b413a53992:33601 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T16:17:15,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T16:17:15,336 INFO [M:0;16b413a53992:33601 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T16:17:15,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33601-0x1002f9f05730000, quorum=127.0.0.1:57646, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T16:17:15,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7740a90{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T16:17:15,350 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c91b148{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T16:17:15,350 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T16:17:15,350 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43a5eb8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T16:17:15,350 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70efab76{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/hadoop.log.dir/,STOPPED} 2024-11-11T16:17:15,357 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T16:17:15,357 WARN [BP-856990112-172.17.0.2-1731341643287 heartbeating to localhost/127.0.0.1:33383 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T16:17:15,357 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T16:17:15,357 WARN [BP-856990112-172.17.0.2-1731341643287 heartbeating to localhost/127.0.0.1:33383 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-856990112-172.17.0.2-1731341643287 (Datanode Uuid 5299dada-c81c-4e5e-99e5-4d475f2a8702) service to localhost/127.0.0.1:33383 2024-11-11T16:17:15,360 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/data/data1/current/BP-856990112-172.17.0.2-1731341643287 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T16:17:15,361 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/cluster_e0888911-68ce-9048-7b55-8279f36b0d35/data/data2/current/BP-856990112-172.17.0.2-1731341643287 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T16:17:15,362 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T16:17:15,370 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3042afd9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T16:17:15,371 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7e4ef918{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T16:17:15,371 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T16:17:15,371 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65b5c633{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T16:17:15,371 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69b57b46{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/14bec2db-c6ed-316c-5bb9-3c949a5492c9/hadoop.log.dir/,STOPPED} 2024-11-11T16:17:15,392 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T16:17:15,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down